diff options
Diffstat (limited to 'src')
210 files changed, 7736 insertions, 3358 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 6b81ecc49..174435e7c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -72,6 +72,7 @@ static int main_build_remote(int argc, char * * argv) settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -112,10 +113,14 @@ static int main_build_remote(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -214,7 +219,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep<StringSet>(", ", m.supportedFeatures) % concatStringsSep<StringSet>(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error); std::cerr << "# decline\n"; } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0740ea960..ab51c229d 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -4,6 +4,7 @@ #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" +#include "repl.hh" #include <nlohmann/json.hpp> @@ -121,12 +122,22 @@ ref<EvalState> EvalCommand::getEvalState() ; if (startReplOnEvalErrors) { - evalState->debugRepl = &runRepl; + evalState->debugRepl = &AbstractNixRepl::runSimple; }; } return ref<EvalState>(evalState); } +MixOperateOnOptions::MixOperateOnOptions() +{ + addFlag({ + .longName = "derivation", + .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .category = installablesCategory, + .handler = {&operateOn, OperateOn::Derivation}, + }); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { @@ -208,20 +219,6 @@ void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePath run(store, *storePaths.begin()); } -Strings editorFor(const Path & file, uint32_t line) -{ - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString<Strings>(editor); - if (line > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos || - editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(file); - return args; -} - MixProfile::MixProfile() { addFlag({ diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 3b4b40981..b6d554aab 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -94,12 +94,8 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional<Path> file; std::optional<std::string> expr; - bool readOnlyMode = false; - // FIXME: move this; not all commands (e.g. 'nix run') use it. - OperateOn operateOn = OperateOn::Output; - - SourceExprCommand(bool supportReadOnlyMode = false); + SourceExprCommand(); std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); @@ -114,6 +110,11 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions void completeInstallable(std::string_view prefix); }; +struct MixReadOnlyOption : virtual Args +{ + MixReadOnlyOption(); +}; + /* A command that operates on a list of "installables", which can be store paths, attribute paths, Nix expressions, etc. */ struct InstallablesCommand : virtual Args, SourceExprCommand @@ -139,7 +140,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr<Installable> installable; - InstallableCommand(bool supportReadOnlyMode = false); + InstallableCommand(); void prepare() override; @@ -153,8 +154,15 @@ private: std::string _installable{"."}; }; +struct MixOperateOnOptions : virtual Args +{ + OperateOn operateOn = OperateOn::Output; + + MixOperateOnOptions(); +}; + /* A command that operates on zero or more store paths. */ -struct BuiltPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -227,10 +235,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name) return RegisterCommand(std::move(name), [](){ return make_ref<T>(); }); } -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ -Strings editorFor(const Path & file, uint32_t line); - struct MixProfile : virtual StoreCommand { std::optional<Path> profile; @@ -280,8 +284,4 @@ void printClosureDiff( const StorePath & afterPath, std::string_view indent); - -void runRepl( - ref<EvalState> evalState, - const ValMap & extraEnv); } diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 140ed3b88..908127b4d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -32,7 +32,77 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "Add *path* to the list of locations used to look up `<...>` file names.", + .description = R"( + Add *path* to the Nix search path. The Nix search path is + initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment + variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle + brackets (i.e., `<nixpkgs>`). + + For instance, passing + + ``` + -I /home/eelco/Dev + -I /etc/nixos + ``` + + will cause Nix to look for paths relative to `/home/eelco/Dev` and + `/etc/nixos`, in that order. This is equivalent to setting the + `NIX_PATH` environment variable to + + ``` + /home/eelco/Dev:/etc/nixos + ``` + + It is also possible to match paths against a prefix. For example, + passing + + ``` + -I nixpkgs=/home/eelco/Dev/nixpkgs-branch + -I /etc/nixos + ``` + + will cause Nix to search for `<nixpkgs/path>` in + `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. + + If a path in the Nix search path starts with `http://` or `https://`, + it is interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must consist of a single + top-level directory. For example, passing + + ``` + -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz + ``` + + tells Nix to download and use the current contents of the `master` + branch in the `nixpkgs` repository. + + The URLs of the tarballs from the official `nixos.org` channels + (see [the manual page for `nix-channel`](../nix-channel.md)) can be + abbreviated as `channel:<channel-name>`. For instance, the + following two flags are equivalent: + + ``` + -I nixpkgs=channel:nixos-21.05 + -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz + ``` + + You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the + search path. For instance, + + ``` + -I nixpkgs=flake:nixpkgs + ``` + + specifies that the prefix `nixpkgs` shall refer to the source tree + downloaded from the `nixpkgs` entry in the flake registry. Similarly, + + ``` + -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 + ``` + + makes `<nixpkgs>` refer to a particular branch of the + `NixOS/nixpkgs` repository on GitHub. + )", .category = category, .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} @@ -89,14 +159,25 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, std::string_view s) { - if (isUri(s)) { - return state.store->toRealPath( - fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).first.storePath); - } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (EvalSettings::isPseudoUrl(s)) { + auto storePath = fetchers::downloadTarball( + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath; + return state.store->toRealPath(storePath); + } + + else if (hasPrefix(s, "flake:")) { + settings.requireExperimentalFeature(Xp::Flakes); + auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + return state.store->toRealPath(storePath); + } + + else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p(s.substr(1, s.size() - 2)); return state.findFile(p); - } else + } + + else return absPath(std::string(s)); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc new file mode 100644 index 000000000..f674f32bd --- /dev/null +++ b/src/libcmd/editor-for.cc @@ -0,0 +1,20 @@ +#include "util.hh" +#include "editor-for.hh" + +namespace nix { + +Strings editorFor(const Path & file, uint32_t line) +{ + auto editor = getEnv("EDITOR").value_or("cat"); + auto args = tokenizeString<Strings>(editor); + if (line > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos || + editor.find("kak") != std::string::npos)) + args.push_back(fmt("+%d", line)); + args.push_back(file); + return args; +} + +} diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh new file mode 100644 index 000000000..8fbd08792 --- /dev/null +++ b/src/libcmd/editor-for.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +/* Helper function to generate args that invoke $EDITOR on + filename:lineno. */ +Strings editorFor(const Path & file, uint32_t line); + +} diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc new file mode 100644 index 000000000..d9377f0d6 --- /dev/null +++ b/src/libcmd/installable-attr-path.cc @@ -0,0 +1,109 @@ +#include "globals.hh" +#include "installable-attr-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +InstallableAttrPath::InstallableAttrPath( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec) + : InstallableValue(state) + , cmd(cmd) + , v(allocRootValue(v)) + , attrPath(attrPath) + , extendedOutputsSpec(std::move(extendedOutputsSpec)) +{ } + +std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state) +{ + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes, pos); + return {vRes, pos}; +} + +DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() +{ + auto v = toValue(*state).first; + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map<StorePath, OutputsSpec> byDrvPath; + + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); + + auto newOutputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + for (auto & output : drvInfo.queryOutputs(false, true)) + outputsToInstall.insert(output.first); + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()); + + auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); + + if (!didInsert) + iter->second = iter->second.union_(newOutputs); + } + + DerivedPathsWithInfo res; + for (auto & [drvPath, outputs] : byDrvPath) + res.push_back({ + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = outputs, + }, + }); + + return res; +} + +InstallableAttrPath InstallableAttrPath::parse( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + return { + state, cmd, v, + prefix == "." ? "" : std::string { prefix }, + extendedOutputsSpec + }; +} + +} diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh new file mode 100644 index 000000000..c06132ec8 --- /dev/null +++ b/src/libcmd/installable-attr-path.hh @@ -0,0 +1,56 @@ +#include "globals.hh" +#include "installable-value.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +class InstallableAttrPath : public InstallableValue +{ + SourceExprCommand & cmd; + RootValue v; + std::string attrPath; + ExtendedOutputsSpec extendedOutputsSpec; + + InstallableAttrPath( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec); + + std::string what() const override { return attrPath; }; + + std::pair<Value *, PosIdx> toValue(EvalState & state) override; + + DerivedPathsWithInfo toDerivedPaths() override; + +public: + + static InstallableAttrPath parse( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc new file mode 100644 index 000000000..a9921b901 --- /dev/null +++ b/src/libcmd/installable-derived-path.cc @@ -0,0 +1,70 @@ +#include "installable-derived-path.hh" +#include "derivations.hh" + +namespace nix { + +std::string InstallableDerivedPath::what() const +{ + return derivedPath.to_string(*store); +} + +DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() +{ + return {{.path = derivedPath, .info = {} }}; +} + +std::optional<StorePath> InstallableDerivedPath::getStorePath() +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + [&](const DerivedPath::Opaque & bo) { + return bo.path; + }, + }, derivedPath.raw()); +} + +InstallableDerivedPath InstallableDerivedPath::parse( + ref<Store> store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more liberally. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + // First, we accept a symlink chain or an actual store path. + auto storePath = store->followLinksToStorePath(prefix); + // Second, we see if the store path ends in `.drv` to decide what sort + // of derived path they want. + // + // This handling predates the `^` syntax. The `^*` in + // `/nix/store/hash-foo.drv^*` unambiguously means "do the + // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could + // also unambiguously mean "do the DerivedPath::Opaque` case". + // + // Issue #7261 tracks reconsidering this `.drv` dispatching. + return storePath.isDerivation() + ? (DerivedPath) DerivedPath::Built { + .drvPath = std::move(storePath), + .outputs = OutputsSpec::All {}, + } + : (DerivedPath) DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + return DerivedPath::Built { + .drvPath = store->parseStorePath(prefix), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw()); + return InstallableDerivedPath { + store, + std::move(derivedPath), + }; +} + +} diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh new file mode 100644 index 000000000..042878b91 --- /dev/null +++ b/src/libcmd/installable-derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "installables.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref<Store> store; + DerivedPath derivedPath; + + InstallableDerivedPath(ref<Store> store, DerivedPath && derivedPath) + : store(store), derivedPath(std::move(derivedPath)) + { } + + std::string what() const override; + + DerivedPathsWithInfo toDerivedPaths() override; + + std::optional<StorePath> getStorePath() override; + + static InstallableDerivedPath parse( + ref<Store> store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc new file mode 100644 index 000000000..60a97deaf --- /dev/null +++ b/src/libcmd/installable-flake.cc @@ -0,0 +1,236 @@ +#include "globals.hh" +#include "installable-flake.hh" +#include "installable-derived-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +std::vector<std::string> InstallableFlake::getActualAttrPaths() +{ + std::vector<std::string> res; + + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); + + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); + + return aOutputs->value; +} + +static std::string showAttrPaths(const std::vector<std::string> & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; + } + return s; +} + +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref<EvalState> state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), + prefixes(fragment == "" ? Strings{} : prefixes), + extendedOutputsSpec(std::move(extendedOutputsSpec)), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() +{ + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + + auto attr = getCursor(*state); + + auto attrPath = attr->getAttrPathStr(); + + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (v.type() == nPath) { + PathSet context; + auto storePath = state->copyPathToStore(context, Path(v.path)); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + } + }}; + } + + else if (v.type() == nString) { + PathSet context; + auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); + auto storePath = state->store->maybeParseStorePath(s); + if (storePath && context.count(std::string(s))) { + return {{ + .path = DerivedPath::Opaque { + .path = std::move(*storePath), + } + }}; + } else + throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); + } + + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); + } + + auto drvPath = attr->forceDerivation(); + + std::optional<NixInt> priority; + + if (attr->maybeGetAttr(state->sOutputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } + + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (aOutputSpecified->getBool()) { + if (auto aOutputName = attr->maybeGetAttr("outputName")) + outputsToInstall = { aOutputName->getString() }; + } + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) + for (auto & s : aOutputsToInstall->getListOfStrings()) + outputsToInstall.insert(s); + } + + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); + + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()), + }, + .info = { + .priority = priority, + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + } + }}; +} + +std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) +{ + return {&getCursor(state)->forceValue(), noPos}; +} + +std::vector<ref<eval_cache::AttrCursor>> +InstallableFlake::getCursors(EvalState & state) +{ + auto evalCache = openEvalCache(state, + std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags))); + + auto root = evalCache->getRoot(); + + std::vector<ref<eval_cache::AttrCursor>> res; + + Suggestions suggestions; + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); + } + } + + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; +} + +std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; + _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + } + return _lockedFlake; +} + +FlakeRef InstallableFlake::nixpkgsFlakeRef() const +{ + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } + } + + return Installable::nixpkgsFlakeRef(); +} + +} diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh new file mode 100644 index 000000000..c75765086 --- /dev/null +++ b/src/libcmd/installable-flake.hh @@ -0,0 +1,50 @@ +#pragma once + +#include "installable-value.hh" + +namespace nix { + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + ExtendedOutputsSpec extendedOutputsSpec; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; + + InstallableFlake( + SourceExprCommand * cmd, + ref<EvalState> state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags); + + std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector<std::string> getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + DerivedPathsWithInfo toDerivedPaths() override; + + std::pair<Value *, PosIdx> toValue(EvalState & state) override; + + /* Get a cursor to every attrpath in getActualAttrPaths() + that exists. However if none exists, throw an exception. */ + std::vector<ref<eval_cache::AttrCursor>> + getCursors(EvalState & state) override; + + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const override; +}; + +ref<eval_cache::EvalCache> openEvalCache( + EvalState & state, + std::shared_ptr<flake::LockedFlake> lockedFlake); + +} diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh new file mode 100644 index 000000000..c6cdc4797 --- /dev/null +++ b/src/libcmd/installable-value.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "installables.hh" + +namespace nix { + +struct InstallableValue : Installable +{ + ref<EvalState> state; + + InstallableValue(ref<EvalState> state) : state(state) {} +}; + +} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index dbe4a449d..00c6f9516 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,5 +1,9 @@ #include "globals.hh" #include "installables.hh" +#include "installable-derived-path.hh" +#include "installable-attr-path.hh" +#include "installable-flake.hh" +#include "outputs-spec.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -143,7 +147,7 @@ void MixFlakeOptions::completionHook() completeFlakeInput(*prefix); } -SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) +SourceExprCommand::SourceExprCommand() { addFlag({ .longName = "file", @@ -165,24 +169,18 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) .labels = {"expr"}, .handler = {&expr} }); +} +MixReadOnlyOption::MixReadOnlyOption() +{ addFlag({ - .longName = "derivation", - .description = "Operate on the store derivation rather than its outputs.", - .category = installablesCategory, - .handler = {&operateOn, OperateOn::Derivation}, + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&settings.readOnlyMode, true}, }); - - if (supportReadOnlyMode) { - addFlag({ - .longName = "read-only", - .description = - "Do not instantiate each evaluated derivation. " - "This improves performance, but can cause errors when accessing " - "store paths of derivations during evaluation.", - .handler = {&readOnlyMode, true}, - }); - } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -358,7 +356,7 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix) } } -DerivedPath Installable::toDerivedPath() +DerivedPathWithInfo Installable::toDerivedPath() { auto buildables = toDerivedPaths(); if (buildables.size() != 1) @@ -378,10 +376,9 @@ Installable::getCursors(EvalState & state) ref<eval_cache::AttrCursor> Installable::getCursor(EvalState & state) { - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); } static StorePath getDeriver( @@ -396,169 +393,6 @@ static StorePath getDeriver( return *derivers.begin(); } -struct InstallableStorePath : Installable -{ - ref<Store> store; - StorePath storePath; - - InstallableStorePath(ref<Store> store, StorePath && storePath) - : store(store), storePath(std::move(storePath)) { } - - std::string what() const override { return store->printStorePath(storePath); } - - DerivedPaths toDerivedPaths() override - { - if (storePath.isDerivation()) { - auto drv = store->readDerivation(storePath); - return { - DerivedPath::Built { - .drvPath = storePath, - .outputs = drv.outputNames(), - } - }; - } else { - return { - DerivedPath::Opaque { - .path = storePath, - } - }; - } - } - - StorePathSet toDrvPaths(ref<Store> store) override - { - if (storePath.isDerivation()) { - return {storePath}; - } else { - return {getDeriver(store, *this, storePath)}; - } - } - - std::optional<StorePath> getStorePath() override - { - return storePath; - } -}; - -DerivedPaths InstallableValue::toDerivedPaths() -{ - DerivedPaths res; - - std::map<StorePath, std::set<std::string>> drvsToOutputs; - RealisedPath::Set drvsToCopy; - - // Group by derivation, helps with .all in particular - for (auto & drv : toDerivations()) { - for (auto & outputName : drv.outputsToInstall) - drvsToOutputs[drv.drvPath].insert(outputName); - drvsToCopy.insert(drv.drvPath); - } - - for (auto & i : drvsToOutputs) - res.push_back(DerivedPath::Built { i.first, i.second }); - - return res; -} - -StorePathSet InstallableValue::toDrvPaths(ref<Store> store) -{ - StorePathSet res; - for (auto & drv : toDerivations()) - res.insert(drv.drvPath); - return res; -} - -struct InstallableAttrPath : InstallableValue -{ - SourceExprCommand & cmd; - RootValue v; - std::string attrPath; - OutputsSpec outputsSpec; - - InstallableAttrPath( - ref<EvalState> state, - SourceExprCommand & cmd, - Value * v, - const std::string & attrPath, - OutputsSpec outputsSpec) - : InstallableValue(state) - , cmd(cmd) - , v(allocRootValue(v)) - , attrPath(attrPath) - , outputsSpec(std::move(outputsSpec)) - { } - - std::string what() const override { return attrPath; } - - std::pair<Value *, PosIdx> toValue(EvalState & state) override - { - auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); - state.forceValue(*vRes, pos); - return {vRes, pos}; - } - - virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; -}; - -std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations() -{ - auto v = toValue(*state).first; - - Bindings & autoArgs = *cmd.getAutoArgs(*state); - - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); - - std::vector<DerivationInfo> res; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); - - std::set<std::string> outputsToInstall; - - if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - outputsToInstall = *outputNames; - else - for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec))) - outputsToInstall.insert(output.first); - - res.push_back(DerivationInfo { - .drvPath = *drvPath, - .outputsToInstall = std::move(outputsToInstall) - }); - } - - return res; -} - -std::vector<std::string> InstallableFlake::getActualAttrPaths() -{ - std::vector<std::string> res; - - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); - - for (auto & s : attrPaths) - res.push_back(s); - - return res; -} - -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); - - return aOutputs->value; -} - ref<eval_cache::EvalCache> openEvalCache( EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake) @@ -579,7 +413,7 @@ ref<eval_cache::EvalCache> openEvalCache( auto vFlake = state.allocValue(); flake::callFlake(state, *lockedFlake, *vFlake); - state.forceAttrs(*vFlake, noPos); + state.forceAttrs(*vFlake, noPos, "while parsing cached flake data"); auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); assert(aOutputs); @@ -588,186 +422,11 @@ ref<eval_cache::EvalCache> openEvalCache( }); } -static std::string showAttrPaths(const std::vector<std::string> & paths) -{ - std::string s; - for (const auto & [n, i] : enumerate(paths)) { - if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; - s += '\''; s += i; s += '\''; - } - return s; -} - -InstallableFlake::InstallableFlake( - SourceExprCommand * cmd, - ref<EvalState> state, - FlakeRef && flakeRef, - std::string_view fragment, - OutputsSpec outputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags) - : InstallableValue(state), - flakeRef(flakeRef), - attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), - prefixes(fragment == "" ? Strings{} : prefixes), - outputsSpec(std::move(outputsSpec)), - lockFlags(lockFlags) -{ - if (cmd && cmd->getAutoArgs(*state)->size()) - throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); -} - -std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() -{ - Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); - - auto attr = getCursor(*state); - - auto attrPath = attr->getAttrPathStr(); - - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); - - auto drvPath = attr->forceDerivation(); - - std::set<std::string> outputsToInstall; - std::optional<NixInt> priority; - - if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { - if (aOutputSpecified->getBool()) { - if (auto aOutputName = attr->maybeGetAttr("outputName")) - outputsToInstall = { aOutputName->getString() }; - } - } - - else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) - for (auto & s : aOutputsToInstall->getListOfStrings()) - outputsToInstall.insert(s); - if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); - } - - if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) { - outputsToInstall.clear(); - if (auto aOutputs = attr->maybeGetAttr(state->sOutputs)) - for (auto & s : aOutputs->getListOfStrings()) - outputsToInstall.insert(s); - } - - if (outputsToInstall.empty()) - outputsToInstall.insert("out"); - - if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - outputsToInstall = *outputNames; - - auto drvInfo = DerivationInfo { - .drvPath = std::move(drvPath), - .outputsToInstall = std::move(outputsToInstall), - .priority = priority, - }; - - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; -} - -std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() -{ - std::vector<DerivationInfo> res; - res.push_back(std::get<2>(toDerivation())); - return res; -} - -std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) -{ - return {&getCursor(state)->forceValue(), noPos}; -} - -std::vector<ref<eval_cache::AttrCursor>> -InstallableFlake::getCursors(EvalState & state) -{ - auto evalCache = openEvalCache(state, - std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - std::vector<ref<eval_cache::AttrCursor>> res; - - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back(ref(*attr)); - } - - return res; -} - -ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state) -{ - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(state, lockedFlake); - auto root = cache->getRoot(); - - Suggestions suggestions; - - auto attrPaths = getActualAttrPaths(); - - for (auto & attrPath : attrPaths) { - debug("trying flake output attribute '%s'", attrPath); - - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } - - return *attrOrSuggestions; - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); -} - -std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const -{ - if (!_lockedFlake) { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); - } - return _lockedFlake; -} - -FlakeRef InstallableFlake::nixpkgsFlakeRef() const -{ - auto lockedFlake = getLockedFlake(); - - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } - } - - return Installable::nixpkgsFlakeRef(); -} - std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( ref<Store> store, std::vector<std::string> ss) { std::vector<std::shared_ptr<Installable>> result; - if (readOnlyMode) { - settings.readOnlyMode = true; - } - if (file || expr) { if (file && expr) throw UsageError("'--file' and '--expr' are exclusive"); @@ -781,7 +440,8 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( if (file == "-") { auto e = state->parseStdin(); state->eval(e, *vFile); - } else if (file) + } + else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { auto e = state->parseExprFromString(*expr, absPath(".")); @@ -789,12 +449,11 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( } for (auto & s : ss) { - auto [prefix, outputsSpec] = parseOutputsSpec(s); + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s); result.push_back( std::make_shared<InstallableAttrPath>( - state, *this, vFile, - prefix == "." ? "" : prefix, - outputsSpec)); + InstallableAttrPath::parse( + state, *this, vFile, prefix, extendedOutputsSpec))); } } else { @@ -802,9 +461,15 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( for (auto & s : ss) { std::exception_ptr ex; - if (s.find('/') != std::string::npos) { + auto [prefix_, extendedOutputsSpec_] = ExtendedOutputsSpec::parse(s); + // To avoid clang's pedantry + auto prefix = std::move(prefix_); + auto extendedOutputsSpec = std::move(extendedOutputsSpec_); + + if (prefix.find('/') != std::string::npos) { try { - result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s))); + result.push_back(std::make_shared<InstallableDerivedPath>( + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); continue; } catch (BadStorePath &) { } catch (...) { @@ -814,13 +479,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); result.push_back(std::make_shared<InstallableFlake>( this, getEvalState(), std::move(flakeRef), fragment, - outputsSpec, + extendedOutputsSpec, getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); @@ -867,13 +532,19 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal if (mode == Realise::Nothing) settings.readOnlyMode = true; + struct Aux + { + ExtraPathInfo info; + std::shared_ptr<Installable> installable; + }; + std::vector<DerivedPath> pathsToBuild; - std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap; + std::map<DerivedPath, std::vector<Aux>> backmap; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b); - backmap[b].push_back(i); + pathsToBuild.push_back(b.path); + backmap[b.path].push_back({.info = b.info, .installable = i}); } } @@ -886,42 +557,18 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal printMissing(store, pathsToBuild, lvlError); for (auto & path : pathsToBuild) { - for (auto & installable : backmap[path]) { + for (auto & aux : backmap[path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store->queryRealisation(outputId); - if (!realisation) - throw Error( - "cannot operate on an output of the " - "unbuilt derivation '%s'", - outputId.to_string()); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign( - output, *drvOutput->second); - } - } - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); + auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info}}); }, }, path.raw()); } @@ -937,16 +584,22 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal if (!buildResult.success()) buildResult.rethrow(); - for (auto & installable : backmap[buildResult.path]) { + for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map<std::string, StorePath> outputs; for (auto & path : buildResult.builtOutputs) outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info, + .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info, + .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -1031,7 +684,7 @@ StorePathSet Installable::toDerivations( [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); return drvPaths; } @@ -1052,8 +705,8 @@ void InstallablesCommand::prepare() installables = load(); } -Installables InstallablesCommand::load() { - Installables installables; +Installables InstallablesCommand::load() +{ if (_installables.empty() && useDefaultInstallables()) // FIXME: commands like "nix profile install" should not have a // default, probably. @@ -1063,16 +716,13 @@ Installables InstallablesCommand::load() { std::vector<std::string> InstallablesCommand::getFlakesForCompletion() { - if (_installables.empty()) { - if (useDefaultInstallables()) - return {"."}; - return {}; - } + if (_installables.empty() && useDefaultInstallables()) + return {"."}; return _installables; } -InstallableCommand::InstallableCommand(bool supportReadOnlyMode) - : SourceExprCommand(supportReadOnlyMode) +InstallableCommand::InstallableCommand() + : SourceExprCommand() { expectArgs({ .label = "installable", diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 02ea351d3..be77fdc81 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,7 +2,7 @@ #include "util.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derived-path.hh" #include "eval.hh" #include "store-api.hh" @@ -20,7 +20,7 @@ namespace eval_cache { class EvalCache; class AttrCursor; } struct App { - std::vector<StorePathWithOutputs> context; + std::vector<DerivedPath> context; Path program; // FIXME: add args, sandbox settings, metadata, ... }; @@ -52,26 +52,42 @@ enum class OperateOn { Derivation }; +struct ExtraPathInfo +{ + std::optional<NixInt> priority; + std::optional<FlakeRef> originalRef; + std::optional<FlakeRef> resolvedRef; + std::optional<std::string> attrPath; + // FIXME: merge with DerivedPath's 'outputs' field? + std::optional<ExtendedOutputsSpec> extendedOutputsSpec; +}; + +/* A derived path with any additional info that commands might + need from the derivation. */ +struct DerivedPathWithInfo +{ + DerivedPath path; + ExtraPathInfo info; +}; + struct BuiltPathWithResult { BuiltPath path; + ExtraPathInfo info; std::optional<BuildResult> result; }; +typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo; + struct Installable { virtual ~Installable() { } virtual std::string what() const = 0; - virtual DerivedPaths toDerivedPaths() = 0; - - virtual StorePathSet toDrvPaths(ref<Store> store) - { - throw Error("'%s' cannot be converted to a derivation path", what()); - } + virtual DerivedPathsWithInfo toDerivedPaths() = 0; - DerivedPath toDerivedPath(); + DerivedPathWithInfo toDerivedPath(); UnresolvedApp toApp(EvalState & state); @@ -87,9 +103,13 @@ struct Installable return {}; } + /* Get a cursor to each value this Installable could refer to. However + if none exists, throw exception instead of returning empty vector. */ virtual std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state); + /* Get the first and most preferred cursor this Installable could refer + to, or throw an exception if none exists. */ virtual ref<eval_cache::AttrCursor> getCursor(EvalState & state); @@ -141,73 +161,4 @@ struct Installable typedef std::vector<std::shared_ptr<Installable>> Installables; -struct InstallableValue : Installable -{ - ref<EvalState> state; - - InstallableValue(ref<EvalState> state) : state(state) {} - - struct DerivationInfo - { - StorePath drvPath; - std::set<std::string> outputsToInstall; - std::optional<NixInt> priority; - }; - - virtual std::vector<DerivationInfo> toDerivations() = 0; - - DerivedPaths toDerivedPaths() override; - - StorePathSet toDrvPaths(ref<Store> store) override; -}; - -struct InstallableFlake : InstallableValue -{ - FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; - OutputsSpec outputsSpec; - const flake::LockFlags & lockFlags; - mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; - - InstallableFlake( - SourceExprCommand * cmd, - ref<EvalState> state, - FlakeRef && flakeRef, - std::string_view fragment, - OutputsSpec outputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags); - - std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } - - std::vector<std::string> getActualAttrPaths(); - - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - - std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation(); - - std::vector<DerivationInfo> toDerivations() override; - - std::pair<Value *, PosIdx> toValue(EvalState & state) override; - - /* Get a cursor to every attrpath in getActualAttrPaths() that - exists. */ - std::vector<ref<eval_cache::AttrCursor>> - getCursors(EvalState & state) override; - - /* Get a cursor to the first attrpath in getActualAttrPaths() that - exists, or throw an exception with suggestions if none exists. */ - ref<eval_cache::AttrCursor> getCursor(EvalState & state) override; - - std::shared_ptr<flake::LockedFlake> getLockedFlake() const; - - FlakeRef nixpkgsFlakeRef() const override; -}; - -ref<eval_cache::EvalCache> openEvalCache( - EvalState & state, - std::shared_ptr<flake::LockedFlake> lockedFlake); - } diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 152bc388d..541a7d2ba 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,7 +6,7 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix +libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in index 1761a9f41..39575f222 100644 --- a/src/libcmd/nix-cmd.pc.in +++ b/src/libcmd/nix-cmd.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixcmd -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 557952277..e3afb1531 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -19,6 +19,8 @@ extern "C" { } #endif +#include "repl.hh" + #include "ansicolor.hh" #include "shared.hh" #include "eval.hh" @@ -31,7 +33,9 @@ extern "C" { #include "get-drvs.hh" #include "derivations.hh" #include "globals.hh" -#include "command.hh" +#include "flake/flake.hh" +#include "flake/lockfile.hh" +#include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" @@ -45,18 +49,16 @@ extern "C" { namespace nix { struct NixRepl + : AbstractNixRepl #if HAVE_BOEHMGC - : gc + , gc #endif { std::string curDir; - ref<EvalState> state; - Bindings * autoArgs; size_t debugTraceIndex; Strings loadedFiles; - typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; std::function<AnnotatedValues()> getValues; const static int envSize = 32768; @@ -69,8 +71,11 @@ struct NixRepl NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state, std::function<AnnotatedValues()> getValues); - ~NixRepl(); - void mainLoop(); + virtual ~NixRepl(); + + void mainLoop() override; + void initEnv() override; + StringSet completePrefix(const std::string & prefix); bool getLine(std::string & input, const std::string & prompt); StorePath getDerivationPath(Value & v); @@ -78,7 +83,6 @@ struct NixRepl void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); - void initEnv(); void loadFiles(); void reloadFiles(); void addAttrsToScope(Value & attrs); @@ -92,7 +96,6 @@ struct NixRepl std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); }; - std::string removeWhitespace(std::string s) { s = chomp(s); @@ -104,7 +107,7 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, std::function<NixRepl::AnnotatedValues()> getValues) - : state(state) + : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(false, state->staticBaseEnv.get())) @@ -215,17 +218,15 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi out << dt.hint.str() << "\n"; // prefer direct pos, but if noPos then try the expr. - auto pos = *dt.pos - ? *dt.pos - : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]; + auto pos = dt.pos + ? dt.pos + : static_cast<std::shared_ptr<AbstractPos>>(positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]); if (pos) { - printAtPos(pos, out); - - auto loc = getCodeLines(pos); - if (loc.has_value()) { + out << pos; + if (auto loc = pos->getCodeLines()) { out << "\n"; - printCodeLines(out, "", pos, *loc); + printCodeLines(out, "", *pos, *loc); out << "\n"; } } @@ -399,7 +400,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) Expr * e = parseString(expr); Value v; e->eval(*state, *env, v); - state->forceAttrs(v, noPos); + state->forceAttrs(v, noPos, "while evaluating an attrset for the purpose of completion (this error should not be displayed; file an issue?)"); for (auto & i : *v.attrs) { std::string_view name = state->symbols[i.name]; @@ -589,15 +590,17 @@ bool NixRepl::processLine(std::string line) Value v; evalString(arg, v); - const auto [file, line] = [&] () -> std::pair<std::string, uint32_t> { + const auto [path, line] = [&] () -> std::pair<Path, uint32_t> { if (v.type() == nPath || v.type() == nString) { PathSet context; - auto filename = state->coerceToString(noPos, v, context).toOwned(); - state->symbols.create(filename); - return {filename, 0}; + auto path = state->coerceToPath(noPos, v, context, "while evaluating the filename to edit"); + return {path, 0}; } else if (v.isLambda()) { auto pos = state->positions[v.lambda.fun->pos]; - return {pos.file, pos.line}; + if (auto path = std::get_if<Path>(&pos.origin)) + return {*path, pos.line}; + else + throw Error("'%s' cannot be shown in an editor", pos); } else { // assume it's a derivation return findPackageFilename(*state, v, arg); @@ -605,7 +608,7 @@ bool NixRepl::processLine(std::string line) }(); // Open in EDITOR - auto args = editorFor(file, line); + auto args = editorFor(path, line); auto editor = args.front(); args.pop_front(); @@ -641,7 +644,12 @@ bool NixRepl::processLine(std::string line) Path drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b" || command == ":bl") { - state->store->buildPaths({DerivedPath::Built{drvPath}}); + state->store->buildPaths({ + DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { @@ -787,7 +795,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, - .allowMutable = !evalSettings.pureEval, + .allowUnlocked = !evalSettings.pureEval, }), v); addAttrsToScope(v); @@ -834,7 +842,7 @@ void NixRepl::loadFiles() void NixRepl::addAttrsToScope(Value & attrs) { - state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }); + state->forceAttrs(attrs, [&]() { return attrs.determinePos(noPos); }, "while evaluating an attribute set to be merged in the global scope"); if (displ + attrs.attrs->size() >= envSize) throw Error("environment full; cannot add more variables"); @@ -939,7 +947,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m Bindings::iterator i = v.attrs->find(state->sDrvPath); PathSet context; if (i != v.attrs->end()) - str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context)); + str << state->store->printStorePath(state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); else str << "???"; str << "»"; @@ -1024,8 +1032,22 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m return str; } -void runRepl( - ref<EvalState>evalState, + +std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create( + const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + std::function<AnnotatedValues()> getValues) +{ + return std::make_unique<NixRepl>( + searchPath, + openStore(), + state, + getValues + ); +} + + +void AbstractNixRepl::runSimple( + ref<EvalState> evalState, const ValMap & extraEnv) { auto getValues = [&]()->NixRepl::AnnotatedValues{ @@ -1049,91 +1071,4 @@ void runRepl( repl->mainLoop(); } -struct CmdRepl : InstallablesCommand -{ - CmdRepl() { - evalSettings.pureEval = false; - } - - void prepare() override - { - if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { - warn("future versions of Nix will require using `--file` to load a file"); - if (this->_installables.size() > 1) - warn("more than one input file is not currently supported"); - auto filePath = this->_installables[0].data(); - file = std::optional(filePath); - _installables.front() = _installables.back(); - _installables.pop_back(); - } - installables = InstallablesCommand::load(); - } - - std::vector<std::string> files; - - Strings getDefaultFlakeAttrPaths() override - { - return {""}; - } - - bool useDefaultInstallables() override - { - return file.has_value() or expr.has_value(); - } - - bool forceImpureByDefault() override - { - return true; - } - - std::string description() override - { - return "start an interactive environment for evaluating Nix expressions"; - } - - std::string doc() override - { - return - #include "repl.md" - ; - } - - void run(ref<Store> store) override - { - auto state = getEvalState(); - auto getValues = [&]()->NixRepl::AnnotatedValues{ - auto installables = load(); - NixRepl::AnnotatedValues values; - for (auto & installable: installables){ - auto what = installable->what(); - if (file){ - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); - state->forceValue(*val, pos); - auto autoArgs = getAutoArgs(*state); - auto valPost = state->allocValue(); - state->autoCallFunction(*autoArgs, *val, *valPost); - state->forceValue(*valPost, pos); - values.push_back( {valPost, what }); - } else { - auto [val, pos] = installable->toValue(*state); - values.push_back( {val, what} ); - } - } - return values; - }; - auto repl = std::make_unique<NixRepl>( - searchPath, - openStore(), - state, - getValues - ); - repl->autoArgs = getAutoArgs(*repl->state); - repl->initEnv(); - repl->mainLoop(); - } -}; - -static auto rCmdRepl = registerCommand<CmdRepl>("repl"); - } diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh new file mode 100644 index 000000000..dfccc93e7 --- /dev/null +++ b/src/libcmd/repl.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "eval.hh" + +#if HAVE_BOEHMGC +#define GC_INCLUDE_NEW +#include <gc/gc_cpp.h> +#endif + +namespace nix { + +struct AbstractNixRepl +{ + ref<EvalState> state; + Bindings * autoArgs; + + AbstractNixRepl(ref<EvalState> state) + : state(state) + { } + + virtual ~AbstractNixRepl() + { } + + typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; + + static std::unique_ptr<AbstractNixRepl> create( + const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + std::function<AnnotatedValues()> getValues); + + static void runSimple( + ref<EvalState> evalState, + const ValMap & extraEnv); + + virtual void initEnv() = 0; + + virtual void mainLoop() = 0; +}; + +} diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 94ab60f9a..7c0705091 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -118,7 +118,7 @@ std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & // FIXME: is it possible to extract the Pos object instead of doing this // toString + parsing? - auto pos = state.forceString(*v2); + auto pos = state.forceString(*v2, noPos, "while evaluating the 'meta.position' attribute of a derivation"); auto colon = pos.rfind(':'); if (colon == std::string::npos) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index b259eec63..1219b2471 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(cfg, s)); + context.push_back(NixStringContextElem::parse(cfg, s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -385,7 +385,7 @@ Value & AttrCursor::getValue() if (!_value) { if (parent) { auto & vParent = parent->first->getValue(); - root->state.forceAttrs(vParent, noPos); + root->state.forceAttrs(vParent, noPos, "while searching for an attribute"); auto attr = vParent.attrs->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); @@ -571,14 +571,14 @@ std::string AttrCursor::getString() debug("using cached string attribute '%s'", getAttrPathStr()); return s->first; } else - root->state.debugThrowLastTrace(TypeError("'%s' is not a string", getAttrPathStr())); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow<TypeError>(); } } auto & v = forceValue(); if (v.type() != nString && v.type() != nPath) - root->state.debugThrowLastTrace(TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()))); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>(); return v.type() == nString ? v.string.s : v.path; } @@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if<string_t>(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - if (!root->state.store->isValidPath(c.first)) { + const StorePath & path = std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { + return d.drvPath; + }, + [&](const NixStringContextElem::Built & b) -> const StorePath & { + return b.drvPath; + }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath & { + return o.path; + }, + }, c.raw()); + if (!root->state.store->isValidPath(path)) { valid = false; break; } @@ -602,7 +613,7 @@ string_t AttrCursor::getStringWithContext() return *s; } } else - root->state.debugThrowLastTrace(TypeError("'%s' is not a string", getAttrPathStr())); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow<TypeError>(); } } @@ -613,7 +624,7 @@ string_t AttrCursor::getStringWithContext() else if (v.type() == nPath) return {v.path, {}}; else - root->state.debugThrowLastTrace(TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()))); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>(); } bool AttrCursor::getBool() @@ -626,14 +637,14 @@ bool AttrCursor::getBool() debug("using cached Boolean attribute '%s'", getAttrPathStr()); return *b; } else - root->state.debugThrowLastTrace(TypeError("'%s' is not a Boolean", getAttrPathStr())); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow<TypeError>(); } } auto & v = forceValue(); if (v.type() != nBool) - root->state.debugThrowLastTrace(TypeError("'%s' is not a Boolean", getAttrPathStr())); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow<TypeError>(); return v.boolean; } @@ -645,17 +656,17 @@ NixInt AttrCursor::getInt() cachedValue = root->db->getAttr(getKey()); if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { if (auto i = std::get_if<int_t>(&cachedValue->second)) { - debug("using cached Integer attribute '%s'", getAttrPathStr()); + debug("using cached integer attribute '%s'", getAttrPathStr()); return i->x; } else - throw TypeError("'%s' is not an Integer", getAttrPathStr()); + throw TypeError("'%s' is not an integer", getAttrPathStr()); } } auto & v = forceValue(); if (v.type() != nInt) - throw TypeError("'%s' is not an Integer", getAttrPathStr()); + throw TypeError("'%s' is not an integer", getAttrPathStr()); return v.integer; } @@ -685,7 +696,7 @@ std::vector<std::string> AttrCursor::getListOfStrings() std::vector<std::string> res; for (auto & elem : v.listItems()) - res.push_back(std::string(root->state.forceStringNoCtx(*elem))); + res.push_back(std::string(root->state.forceStringNoCtx(*elem, noPos, "while evaluating an attribute for caching"))); if (root->db) cachedValue = {root->db->setListOfStrings(getKey(), res), res}; @@ -703,14 +714,14 @@ std::vector<Symbol> AttrCursor::getAttrs() debug("using cached attrset attribute '%s'", getAttrPathStr()); return *attrs; } else - root->state.debugThrowLastTrace(TypeError("'%s' is not an attribute set", getAttrPathStr())); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow<TypeError>(); } } auto & v = forceValue(); if (v.type() != nAttrs) - root->state.debugThrowLastTrace(TypeError("'%s' is not an attribute set", getAttrPathStr())); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow<TypeError>(); std::vector<Symbol> attrs; for (auto & attr : *getValue().attrs) diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f2f4ba725..f0da688db 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -103,33 +103,36 @@ void EvalState::forceValue(Value & v, Callable getPos) else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.isBlackhole()) - throwEvalError(getPos(), "infinite recursion encountered"); + error("infinite recursion encountered").atPos(getPos()).template debugThrow<EvalError>(); } [[gnu::always_inline]] -inline void EvalState::forceAttrs(Value & v, const PosIdx pos) +inline void EvalState::forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceAttrs(v, [&]() { return pos; }); + forceAttrs(v, [&]() { return pos; }, errorCtx); } template <typename Callable> [[gnu::always_inline]] -inline void EvalState::forceAttrs(Value & v, Callable getPos) +inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx) { - forceValue(v, getPos); - if (v.type() != nAttrs) - throwTypeError(getPos(), "value is %1% while a set was expected", v); + forceValue(v, noPos); + if (v.type() != nAttrs) { + PosIdx pos = getPos(); + error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>(); + } } [[gnu::always_inline]] -inline void EvalState::forceList(Value & v, const PosIdx pos) +inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (!v.isList()) - throwTypeError(pos, "value is %1% while a list was expected", v); + forceValue(v, noPos); + if (!v.isList()) { + error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow<TypeError>(); + } } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 76a10b9f8..3e37c7f60 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -11,7 +11,9 @@ #include <algorithm> #include <chrono> +#include <iostream> #include <cstring> +#include <optional> #include <unistd.h> #include <sys/time.h> #include <sys/resource.h> @@ -45,7 +47,7 @@ static char * allocString(size_t size) #if HAVE_BOEHMGC t = (char *) GC_MALLOC_ATOMIC(size); #else - t = malloc(size); + t = (char *) malloc(size); #endif if (!t) throw std::bad_alloc(); return t; @@ -67,22 +69,19 @@ static char * dupString(const char * s) // When there's no need to write to the string, we can optimize away empty // string allocations. -// This function handles makeImmutableStringWithLen(null, 0) by returning the -// empty string. -static const char * makeImmutableStringWithLen(const char * s, size_t size) +// This function handles makeImmutableString(std::string_view()) by returning +// the empty string. +static const char * makeImmutableString(std::string_view s) { + const size_t size = s.size(); if (size == 0) return ""; auto t = allocString(size + 1); - memcpy(t, s, size); - t[size] = 0; + memcpy(t, s.data(), size); + t[size] = '\0'; return t; } -static inline const char * makeImmutableString(std::string_view s) { - return makeImmutableStringWithLen(s.data(), s.size()); -} - RootValue allocRootValue(Value * v) { @@ -321,7 +320,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } else { Value nameValue; name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue); + state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); return state.symbols.create(nameValue.string.s); } } @@ -402,7 +401,8 @@ static Strings parseNixPath(const std::string & s) } if (*p == ':') { - if (isUri(std::string(start2, s.end()))) { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { ++p; while (p != s.end() && *p != ':') ++p; } @@ -416,6 +416,44 @@ static Strings parseNixPath(const std::string & s) return res; } +ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) +{ + info.errPos = state.positions[pos]; + return *this; +} + +ErrorBuilder & ErrorBuilder::withTrace(PosIdx pos, const std::string_view text) +{ + info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = false }); + return *this; +} + +ErrorBuilder & ErrorBuilder::withFrameTrace(PosIdx pos, const std::string_view text) +{ + info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = true }); + return *this; +} + +ErrorBuilder & ErrorBuilder::withSuggestions(Suggestions & s) +{ + info.suggestions = s; + return *this; +} + +ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) +{ + // NOTE: This is abusing side-effects. + // TODO: check compatibility with nested debugger calls. + state.debugTraces.push_front(DebugTrace { + .pos = nullptr, + .expr = expr, + .env = env, + .hint = hintformat("Fake frame for debugging purposes"), + .isError = true + }); + return *this; +} + EvalState::EvalState( const Strings & _searchPath, @@ -470,9 +508,6 @@ EvalState::EvalState( #if HAVE_BOEHMGC , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr)) , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr)) -#else - , valueAllocCache(std::make_shared<void *>(nullptr)) - , env1AllocCache(std::make_shared<void *>(nullptr)) #endif , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared<StaticEnv>(false, nullptr)} @@ -484,6 +519,7 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); /* Initialise the Nix expression search path. */ + evalSettings.nixPath.setDefault(evalSettings.getDefaultNixPath()); if (!evalSettings.pureEval) { for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); @@ -651,25 +687,7 @@ void EvalState::addConstant(const std::string & name, Value * v) Value * EvalState::addPrimOp(const std::string & name, size_t arity, PrimOpFun primOp) { - auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - auto sym = symbols.create(name2); - - /* Hack to make constants lazy: turn them into a application of - the primop to a dummy value. */ - if (arity == 0) { - auto vPrimOp = allocValue(); - vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = name2 }); - Value v; - v.mkApp(vPrimOp, vPrimOp); - return addConstant(name, v); - } - - Value * v = allocValue(); - v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = name2 }); - staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; - baseEnv.values[0]->attrs->push_back(Attr(sym, v)); - return v; + return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name }); } @@ -822,7 +840,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & ? std::make_unique<DebugTraceStacker>( *this, DebugTrace { - .pos = error->info().errPos ? *error->info().errPos : positions[expr.getPos()], + .pos = error->info().errPos ? error->info().errPos : static_cast<std::shared_ptr<AbstractPos>>(positions[expr.getPos()]), .expr = expr, .env = env, .hint = error->info().msg, @@ -847,189 +865,27 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & } } -/* Every "format" object (even temporary) takes up a few hundred bytes - of stack space, which is a real killer in the recursive - evaluator. So here are some helper functions for throwing - exceptions. */ -void EvalState::throwEvalError(const PosIdx pos, const char * s, Env & env, Expr & expr) -{ - debugThrow(EvalError({ - .msg = hintfmt(s), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s), - .errPos = positions[pos] - })); -} - -void EvalState::throwEvalError(const char * s, const std::string & s2) -{ - debugThrowLastTrace(EvalError(s, s2)); -} - -void EvalState::throwEvalError(const PosIdx pos, const Suggestions & suggestions, const char * s, - const std::string & s2, Env & env, Expr & expr) -{ - debugThrow(EvalError(ErrorInfo{ - .msg = hintfmt(s, s2), - .errPos = positions[pos], - .suggestions = suggestions, - }), env, expr); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s, s2), - .errPos = positions[pos] - })); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2, Env & env, Expr & expr) -{ - debugThrow(EvalError({ - .msg = hintfmt(s, s2), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwEvalError(const char * s, const std::string & s2, - const std::string & s3) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = positions[noPos] - })); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2, - const std::string & s3) -{ - debugThrowLastTrace(EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = positions[pos] - })); -} - -void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::string & s2, - const std::string & s3, Env & env, Expr & expr) -{ - debugThrow(EvalError({ - .msg = hintfmt(s, s2, s3), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwEvalError(const PosIdx p1, const char * s, const Symbol sym, const PosIdx p2, Env & env, Expr & expr) -{ - // p1 is where the error occurred; p2 is a position mentioned in the message. - debugThrow(EvalError({ - .msg = hintfmt(s, symbols[sym], positions[p2]), - .errPos = positions[p1] - }), env, expr); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s, const Value & v) -{ - debugThrowLastTrace(TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = positions[pos] - })); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s, const Value & v, Env & env, Expr & expr) -{ - debugThrow(TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s) -{ - debugThrowLastTrace(TypeError({ - .msg = hintfmt(s), - .errPos = positions[pos] - })); -} - -void EvalState::throwTypeError(const PosIdx pos, const char * s, const ExprLambda & fun, - const Symbol s2, Env & env, Expr &expr) -{ - debugThrow(TypeError({ - .msg = hintfmt(s, fun.showNamePos(*this), symbols[s2]), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwTypeError(const PosIdx pos, const Suggestions & suggestions, const char * s, - const ExprLambda & fun, const Symbol s2, Env & env, Expr &expr) -{ - debugThrow(TypeError(ErrorInfo { - .msg = hintfmt(s, fun.showNamePos(*this), symbols[s2]), - .errPos = positions[pos], - .suggestions = suggestions, - }), env, expr); -} - -void EvalState::throwTypeError(const char * s, const Value & v, Env & env, Expr &expr) -{ - debugThrow(TypeError({ - .msg = hintfmt(s, showType(v)), - .errPos = positions[expr.getPos()], - }), env, expr); -} - -void EvalState::throwAssertionError(const PosIdx pos, const char * s, const std::string & s1, Env & env, Expr &expr) -{ - debugThrow(AssertionError({ - .msg = hintfmt(s, s1), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwUndefinedVarError(const PosIdx pos, const char * s, const std::string & s1, Env & env, Expr &expr) -{ - debugThrow(UndefinedVarError({ - .msg = hintfmt(s, s1), - .errPos = positions[pos] - }), env, expr); -} - -void EvalState::throwMissingArgumentError(const PosIdx pos, const char * s, const std::string & s1, Env & env, Expr &expr) -{ - debugThrow(MissingArgumentError({ - .msg = hintfmt(s, s1), - .errPos = positions[pos] - }), env, expr); -} - void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const { - e.addTrace(std::nullopt, s, s2); + e.addTrace(nullptr, s, s2); } -void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2) const +void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const { - e.addTrace(positions[pos], s, s2); + e.addTrace(positions[pos], hintfmt(s, s2), frame); } static std::unique_ptr<DebugTraceStacker> makeDebugTraceStacker( EvalState & state, Expr & expr, Env & env, - std::optional<ErrPos> pos, + std::shared_ptr<AbstractPos> && pos, const char * s, const std::string & s2) { return std::make_unique<DebugTraceStacker>(state, DebugTrace { - .pos = pos, + .pos = std::move(pos), .expr = expr, .env = env, .hint = hintfmt(s, s2), @@ -1093,7 +949,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) if (env->type == Env::HasWithExpr) { if (noEval) return 0; Value * v = allocValue(); - evalAttrs(*env->up, (Expr *) env->values[0], *v); + evalAttrs(*env->up, (Expr *) env->values[0], *v, noPos, "<borked>"); env->values[0] = v; env->type = Env::HasWithAttrs; } @@ -1103,7 +959,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) return j->value; } if (!env->prevWith) - throwUndefinedVarError(var.pos, "undefined variable '%1%'", symbols[var.name], *env, const_cast<ExprVar&>(var)); + error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow<UndefinedVarError>(); for (size_t l = env->prevWith; l; --l, env = env->up) ; } } @@ -1135,9 +991,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, PosIdx p) { auto pos = positions[p]; - if (!pos.file.empty()) { + if (auto path = std::get_if<Path>(&pos.origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(pos.file); + attrs.alloc(sFile).mkString(*path); attrs.alloc(sLine).mkInt(pos.line); attrs.alloc(sColumn).mkInt(pos.column); v.mkAttrs(attrs); @@ -1245,7 +1101,7 @@ void EvalState::cacheFile( *this, *e, this->baseEnv, - e->getPos() ? std::optional(ErrPos(positions[e->getPos()])) : std::nullopt, + e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr, "while evaluating the file '%1%':", resolvedPath) : nullptr; @@ -1253,7 +1109,7 @@ void EvalState::cacheFile( // computation. if (mustBeTrivial && !(dynamic_cast<ExprAttrs *>(e))) - throw EvalError("file '%s' must be an attribute set", path); + error("file '%s' must be an attribute set", path).debugThrow<EvalError>(); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); @@ -1271,31 +1127,31 @@ void EvalState::eval(Expr * e, Value & v) } -inline bool EvalState::evalBool(Env & env, Expr * e) -{ - Value v; - e->eval(*this, env, v); - if (v.type() != nBool) - throwTypeError(noPos, "value is %1% while a Boolean was expected", v, env, *e); - return v.boolean; -} - - -inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos) +inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx) { - Value v; - e->eval(*this, env, v); - if (v.type() != nBool) - throwTypeError(pos, "value is %1% while a Boolean was expected", v, env, *e); - return v.boolean; + try { + Value v; + e->eval(*this, env, v); + if (v.type() != nBool) + error("value is %1% while a Boolean was expected", showType(v)).withFrame(env, *e).debugThrow<TypeError>(); + return v.boolean; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) +inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx) { - e->eval(*this, env, v); - if (v.type() != nAttrs) - throwTypeError(noPos, "value is %1% while a set was expected", v, env, *e); + try { + e->eval(*this, env, v); + if (v.type() != nAttrs) + error("value is %1% while a set was expected", showType(v)).withFrame(env, *e).debugThrow<TypeError>(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } @@ -1368,7 +1224,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Hence we need __overrides.) */ if (hasOverrides) { Value * vOverrides = (*v.attrs)[overrides->second.displ].value; - state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }); + state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute"); Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); for (auto & i : *v.attrs) newBnds->push_back(i); @@ -1396,11 +1252,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) state.forceValue(nameVal, i.pos); if (nameVal.type() == nNull) continue; - state.forceStringNoCtx(nameVal); + state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); auto nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - state.throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, j->pos, env, *this); + state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow<EvalError>(); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ @@ -1497,15 +1353,14 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) return; } } else { - state.forceAttrs(*vAttrs, pos); + state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { std::set<std::string> allAttrNames; for (auto & attr : *vAttrs->attrs) allAttrNames.insert(state.symbols[attr.name]); - state.throwEvalError( - pos, - Suggestions::bestMatches(allAttrNames, state.symbols[name]), - "attribute '%1%' missing", state.symbols[name], env, *this); + auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); + state.error("attribute '%1%' missing", state.symbols[name]) + .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow<EvalError>(); } } vAttrs = j->value; @@ -1516,10 +1371,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) ); } catch (Error & e) { - auto pos2r = state.positions[pos2]; - if (pos2 && pos2r.file != state.derivationNixPath) - state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", - showAttrPath(state, env, attrPath)); + if (pos2) { + auto pos2r = state.positions[pos2]; + auto origin = std::get_if<Path>(&pos2r.origin); + if (!(origin && *origin == state.derivationNixPath)) + state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", + showAttrPath(state, env, attrPath)); + } throw; } @@ -1597,7 +1455,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (!lambda.hasFormals()) env2.values[displ++] = args[0]; else { - forceAttrs(*args[0], pos); + try { + forceAttrs(*args[0], lambda.pos, "while evaluating the value passed for the lambda argument"); + } catch (Error & e) { + if (pos) e.addTrace(positions[pos], "from call site"); + throw; + } if (lambda.arg) env2.values[displ++] = args[0]; @@ -1609,8 +1472,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & for (auto & i : lambda.formals->formals) { auto j = args[0]->attrs->get(i.name); if (!j) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name, *fun.lambda.env, lambda); + if (!i.def) { + error("function '%1%' called without required argument '%2%'", + (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), + symbols[i.name]) + .atPos(lambda.pos) + .withTrace(pos, "from call site") + .withFrame(*fun.lambda.env, lambda) + .debugThrow<TypeError>(); + } env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; @@ -1628,11 +1498,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & std::set<std::string> formalNames; for (auto & formal : lambda.formals->formals) formalNames.insert(symbols[formal.name]); - throwTypeError( - pos, - Suggestions::bestMatches(formalNames, symbols[i.name]), - "%1% called with unexpected argument '%2%'", - lambda, i.name, *fun.lambda.env, lambda); + auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); + error("function '%1%' called with unexpected argument '%2%'", + (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), + symbols[i.name]) + .atPos(lambda.pos) + .withTrace(pos, "from call site") + .withSuggestions(suggestions) + .withFrame(*fun.lambda.env, lambda) + .debugThrow<TypeError>(); } abort(); // can't happen } @@ -1655,11 +1529,15 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & lambda.body->eval(*this, env2, vCur); } catch (Error & e) { if (loggerSettings.showTrace.get()) { - addErrorTrace(e, lambda.pos, "while calling %s", - (lambda.name - ? concatStrings("'", symbols[lambda.name], "'") - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); + addErrorTrace( + e, + lambda.pos, + "while calling %s", + lambda.name + ? concatStrings("'", symbols[lambda.name], "'") + : "anonymous lambda", + true); + if (pos) addErrorTrace(e, pos, "from call site%s", "", true); } throw; } @@ -1678,9 +1556,17 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & return; } else { /* We have all the arguments, so call the primop. */ + auto name = vCur.primOp->name; + nrPrimOpCalls++; - if (countCalls) primOpCalls[vCur.primOp->name]++; - vCur.primOp->fun(*this, pos, args, vCur); + if (countCalls) primOpCalls[name]++; + + try { + vCur.primOp->fun(*this, noPos, args, vCur); + } catch (Error & e) { + addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + throw; + } nrArgs -= argsLeft; args += argsLeft; @@ -1715,9 +1601,20 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & for (size_t i = 0; i < argsLeft; ++i) vArgs[argsDone + i] = args[i]; + auto name = primOp->primOp->name; nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, vCur); + if (countCalls) primOpCalls[name]++; + + try { + // TODO: + // 1. Unify this and above code. Heavily redundant. + // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) + // so the debugger allows to inspect the wrong parameters passed to the builtin. + primOp->primOp->fun(*this, noPos, vArgs, vCur); + } catch (Error & e) { + addErrorTrace(e, pos, "while calling the '%1%' builtin", name); + throw; + } nrArgs -= argsLeft; args += argsLeft; @@ -1730,14 +1627,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & heap-allocate a copy and use that instead. */ Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; - /* !!! Should we use the attr pos here? */ - callFunction(*functor->value, 2, args2, vCur, pos); + try { + callFunction(*functor->value, 2, args2, vCur, functor->pos); + } catch (Error & e) { + e.addTrace(positions[pos], "while calling a functor (an attribute set with a '__functor' attribute)"); + throw; + } nrArgs--; args++; } else - throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); + error("attempt to call something which is not a function but %1%", showType(vCur)).atPos(pos).debugThrow<TypeError>(); } vRes = vCur; @@ -1801,13 +1702,12 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { attrs.insert(*j); } else if (!i.def) { - throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') - + error(R"(cannot evaluate a function that has an argument without a value ('%1%') Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See -https://nixos.org/manual/nix/stable/expressions/language-constructs.html#functions.)", symbols[i.name], - *fun.lambda.env, *fun.lambda.fun); +https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name]) + .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow<MissingArgumentError>(); } } } @@ -1830,16 +1730,17 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); + // We cheat in the parser, and pass the position of the condition as the position of the if itself. + (state.evalBool(env, cond, pos, "while evaluating a branch condition") ? then : else_)->eval(state, env, v); } void ExprAssert::eval(EvalState & state, Env & env, Value & v) { - if (!state.evalBool(env, cond, pos)) { + if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { std::ostringstream out; cond->show(state.symbols, out); - state.throwAssertionError(pos, "assertion '%1%' failed", out.str(), env, *this); + state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow<AssertionError>(); } body->eval(state, env, v); } @@ -1847,7 +1748,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e)); + v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: ! } @@ -1855,7 +1756,7 @@ void ExprOpEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - v.mkBool(state.eqValues(v1, v2)); + v.mkBool(state.eqValues(v1, v2, pos, "while testing two values for equality")); } @@ -1863,33 +1764,33 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - v.mkBool(!state.eqValues(v1, v2)); + v.mkBool(!state.eqValues(v1, v2, pos, "while testing two values for inequality")); } void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the AND (&&) operator") && state.evalBool(env, e2, pos, "in the right operand of the AND (&&) operator")); } void ExprOpOr::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos, "in the left operand of the OR (||) operator") || state.evalBool(env, e2, pos, "in the right operand of the OR (||) operator")); } void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(!state.evalBool(env, e1, pos, "in the left operand of the IMPL (->) operator") || state.evalBool(env, e2, pos, "in the right operand of the IMPL (->) operator")); } void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) { Value v1, v2; - state.evalAttrs(env, e1, v1); - state.evalAttrs(env, e2, v2); + state.evalAttrs(env, e1, v1, pos, "in the left operand of the update (//) operator"); + state.evalAttrs(env, e2, v2, pos, "in the right operand of the update (//) operator"); state.nrOpUpdates++; @@ -1928,18 +1829,18 @@ void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); Value * lists[2] = { &v1, &v2 }; - state.concatLists(v, 2, lists, pos); + state.concatLists(v, 2, lists, pos, "while evaluating one of the elements to concatenate"); } -void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos) +void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx) { nrListConcats++; Value * nonEmpty = 0; size_t len = 0; for (size_t n = 0; n < nrLists; ++n) { - forceList(*lists[n], pos); + forceList(*lists[n], pos, errorCtx); auto l = lists[n]->listSize(); len += l; if (l) nonEmpty = lists[n]; @@ -2016,20 +1917,22 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - state.throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp), env, *this); + state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow<EvalError>(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { nf += vTmp.integer; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else - state.throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp), env, *this); + state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow<EvalError>(); } else { if (s.empty()) s.reserve(es->size()); /* skip canonization of first path, which would only be not canonized in the first place if it's coming from a ./${foo} type path */ - auto part = state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first); + auto part = state.coerceToString(i_pos, vTmp, context, + "while evaluating a path segment", + false, firstType == nString, !first); sSize += part->size(); s.emplace_back(std::move(part)); } @@ -2043,7 +1946,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) - state.throwEvalError(pos, "a string that refers to a store path cannot be appended to a path", env, *this); + state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>(); v.mkPath(canonPath(str())); } else v.mkStringMove(c_str(), context); @@ -2093,33 +1996,47 @@ void EvalState::forceValueDeep(Value & v) } -NixInt EvalState::forceInt(Value & v, const PosIdx pos) +NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nInt) - throwTypeError(pos, "value is %1% while an integer was expected", v); - - return v.integer; + try { + forceValue(v, pos); + if (v.type() != nInt) + error("value is %1% while an integer was expected", showType(v)).debugThrow<TypeError>(); + return v.integer; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -NixFloat EvalState::forceFloat(Value & v, const PosIdx pos) +NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() == nInt) - return v.integer; - else if (v.type() != nFloat) - throwTypeError(pos, "value is %1% while a float was expected", v); - return v.fpoint; + try { + forceValue(v, pos); + if (v.type() == nInt) + return v.integer; + else if (v.type() != nFloat) + error("value is %1% while a float was expected", showType(v)).debugThrow<TypeError>(); + return v.fpoint; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } -bool EvalState::forceBool(Value & v, const PosIdx pos) +bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nBool) - throwTypeError(pos, "value is %1% while a Boolean was expected", v); - return v.boolean; + try { + forceValue(v, pos); + if (v.type() != nBool) + error("value is %1% while a Boolean was expected", showType(v)).debugThrow<TypeError>(); + return v.boolean; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } @@ -2129,42 +2046,30 @@ bool EvalState::isFunctor(Value & fun) } -void EvalState::forceFunction(Value & v, const PosIdx pos) -{ - forceValue(v, pos); - if (v.type() != nFunction && !isFunctor(v)) - throwTypeError(pos, "value is %1% while a function was expected", v); -} - - -std::string_view EvalState::forceString(Value & v, const PosIdx pos) +void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, pos); - if (v.type() != nString) { - throwTypeError(pos, "value is %1% while a string was expected", v); + try { + forceValue(v, pos); + if (v.type() != nFunction && !isFunctor(v)) + error("value is %1% while a function was expected", showType(v)).debugThrow<TypeError>(); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; } - return v.string.s; } -/* Decode a context string ‘!<name>!<path>’ into a pair <path, - name>. */ -NixStringContextElem decodeContext(const Store & store, std::string_view s) +std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string_view errorCtx) { - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return { - store.parseStorePath(s.substr(index + 1)), - std::string(s.substr(1, index - 1)), - }; - } else - return { - store.parseStorePath( - s.at(0) == '/' - ? s - : s.substr(1)), - "", - }; + try { + forceValue(v, pos); + if (v.type() != nString) + error("value is %1% while a string was expected", showType(v)).debugThrow<TypeError>(); + return v.string.s; + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } } @@ -2182,29 +2087,24 @@ NixStringContext Value::getContext(const Store & store) assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) - res.push_back(decodeContext(store, *p)); + res.push_back(NixStringContextElem::parse(store, *p)); return res; } -std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos) +std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx) { - auto s = forceString(v, pos); + auto s = forceString(v, pos, errorCtx); copyContext(v, context); return s; } -std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos) +std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { - auto s = forceString(v, pos); + auto s = forceString(v, pos, errorCtx); if (v.string.context) { - if (pos) - throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); - else - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", - v.string.s, v.string.context[0]); + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow<EvalError>(); } return s; } @@ -2228,14 +2128,16 @@ std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & if (i != v.attrs->end()) { Value v1; callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore).toOwned(); + return coerceToString(pos, v1, context, + "while evaluating the result of the `__toString` attribute", + coerceMore, copyToStore).toOwned(); } return {}; } -BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet & context, - bool coerceMore, bool copyToStore, bool canonicalizePath) +BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &context, + std::string_view errorCtx, bool coerceMore, bool copyToStore, bool canonicalizePath) { forceValue(v, pos); @@ -2249,7 +2151,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet if (canonicalizePath) path = canonPath(*path); if (copyToStore) - path = copyPathToStore(context, std::move(path).toOwned()); + path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned())); return path; } @@ -2258,13 +2160,23 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet if (maybeString) return std::move(*maybeString); auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) - throwTypeError(pos, "cannot coerce a set to a string"); - return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + if (i == v.attrs->end()) { + error("cannot coerce %1% to a string", showType(v)) + .withTrace(pos, errorCtx) + .debugThrow<TypeError>(); + } + return coerceToString(pos, *i->value, context, errorCtx, + coerceMore, copyToStore, canonicalizePath); } - if (v.type() == nExternal) - return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore); + if (v.type() == nExternal) { + try { + return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore); + } catch (Error & e) { + e.addTrace(nullptr, errorCtx); + throw; + } + } if (coerceMore) { /* Note that `false' is represented as an empty string for @@ -2278,7 +2190,14 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet if (v.isList()) { std::string result; for (auto [n, v2] : enumerate(v.listItems())) { - result += *coerceToString(pos, *v2, context, coerceMore, copyToStore); + try { + result += *coerceToString(noPos, *v2, context, + "while evaluating one element of the list", + coerceMore, copyToStore, canonicalizePath); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } if (n < v.listSize() - 1 /* !!! not quite correct */ && (!v2->isList() || v2->listSize() != 0)) @@ -2288,56 +2207,55 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet } } - throwTypeError(pos, "cannot coerce %1% to a string", v); + error("cannot coerce %1% to a string", showType(v)) + .withTrace(pos, errorCtx) + .debugThrow<TypeError>(); } -std::string EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) - throwEvalError("file names are not allowed to end in '%1%'", drvExtension); - - Path dstPath; - auto i = srcToStore.find(path); - if (i != srcToStore.end()) - dstPath = store->printStorePath(i->second); - else { - auto p = settings.readOnlyMode + error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>(); + + auto dstPath = [&]() -> StorePath + { + auto i = srcToStore.find(path); + if (i != srcToStore.end()) return i->second; + + auto dstPath = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - dstPath = store->printStorePath(p); - allowPath(p); - srcToStore.insert_or_assign(path, std::move(p)); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); - } + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); - context.insert(dstPath); + context.insert(store->printStorePath(dstPath)); return dstPath; } -Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context) +Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, false, false).toOwned(); + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') - throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>(); return path; } -StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) { - auto path = coerceToString(pos, v, context, false, false).toOwned(); + auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - throw EvalError({ - .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = positions[pos] - }); + error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow<EvalError>(); } -bool EvalState::eqValues(Value & v1, Value & v2) +bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { forceValue(v1, noPos); forceValue(v2, noPos); @@ -2357,7 +2275,6 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (v1.type() != v2.type()) return false; switch (v1.type()) { - case nInt: return v1.integer == v2.integer; @@ -2376,7 +2293,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) - if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; + if (!eqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx)) return false; return true; case nAttrs: { @@ -2386,7 +2303,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) Bindings::iterator i = v1.attrs->find(sOutPath); Bindings::iterator j = v2.attrs->find(sOutPath); if (i != v1.attrs->end() && j != v2.attrs->end()) - return eqValues(*i->value, *j->value); + return eqValues(*i->value, *j->value, pos, errorCtx); } if (v1.attrs->size() != v2.attrs->size()) return false; @@ -2394,7 +2311,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) /* Otherwise, compare the attributes one by one. */ Bindings::iterator i, j; for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) - if (i->name != j->name || !eqValues(*i->value, *j->value)) + if (i->name != j->name || !eqValues(*i->value, *j->value, pos, errorCtx)) return false; return true; @@ -2411,9 +2328,7 @@ bool EvalState::eqValues(Value & v1, Value & v2) return v1.fpoint == v2.fpoint; default: - throwEvalError("cannot compare %1% with %2%", - showType(v1), - showType(v2)); + error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow<EvalError>(); } } @@ -2496,7 +2411,8 @@ void EvalState::printStats() else obj["name"] = nullptr; if (auto pos = positions[fun->pos]) { - obj["file"] = (std::string_view) pos.file; + if (auto path = std::get_if<Path>(&pos.origin)) + obj["file"] = *path; obj["line"] = pos.line; obj["column"] = pos.column; } @@ -2510,7 +2426,8 @@ void EvalState::printStats() for (auto & i : attrSelects) { json obj = json::object(); if (auto pos = positions[i.first]) { - obj["file"] = (const std::string &) pos.file; + if (auto path = std::get_if<Path>(&pos.origin)) + obj["file"] = *path; obj["line"] = pos.line; obj["column"] = pos.column; } @@ -2538,8 +2455,7 @@ void EvalState::printStats() std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string", showType()), - .errPos = pos + .msg = hintfmt("cannot coerce %1% to a string", showType()) }); } @@ -2557,30 +2473,52 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings::EvalSettings() { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); } +/* impure => NIX_PATH or a default path + * restrict-eval => NIX_PATH + * pure-eval => empty + */ Strings EvalSettings::getDefaultNixPath() { - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; + if (pureEval) + return {}; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(getHome() + "/.nix-defexpr/channels"); + auto var = getEnv("NIX_PATH"); + if (var) { + return parseNixPath(*var); + } else if (restrictEval) { + return {}; + } else { + Strings res; + auto add = [&](const Path & p, const std::optional<std::string> & s = std::nullopt) { + if (pathExists(p)) + res.push_back(s ? *s + "=" + p : p); + }; + + add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); add(settings.nixStateDir + "/profiles/per-user/root/channels"); + + return res; } +} - return res; +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); } EvalSettings evalSettings; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f07f15d43..2340ef67b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -60,7 +60,6 @@ void copyContext(const Value & v, PathSet & context); typedef std::map<Path, StorePath> SrcToStore; -std::ostream & printValue(const EvalState & state, std::ostream & str, const Value & v); std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); @@ -78,7 +77,7 @@ struct RegexCache; std::shared_ptr<RegexCache> makeRegexCache(); struct DebugTrace { - std::optional<ErrPos> pos; + std::shared_ptr<AbstractPos> pos; const Expr & expr; const Env & env; hintformat hint; @@ -87,6 +86,43 @@ struct DebugTrace { void debugError(Error * e, Env & env, Expr & expr); +class ErrorBuilder +{ + private: + EvalState & state; + ErrorInfo info; + + ErrorBuilder(EvalState & s, ErrorInfo && i): state(s), info(i) { } + + public: + template<typename... Args> + [[nodiscard, gnu::noinline]] + static ErrorBuilder * create(EvalState & s, const Args & ... args) + { + return new ErrorBuilder(s, ErrorInfo { .msg = hintfmt(args...) }); + } + + [[nodiscard, gnu::noinline]] + ErrorBuilder & atPos(PosIdx pos); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withSuggestions(Suggestions & s); + + [[nodiscard, gnu::noinline]] + ErrorBuilder & withFrame(const Env & e, const Expr & ex); + + template<class ErrorType> + [[gnu::noinline, gnu::noreturn]] + void debugThrow(); +}; + + class EvalState : public std::enable_shared_from_this<EvalState> { public: @@ -146,29 +182,38 @@ public: template<class E> [[gnu::noinline, gnu::noreturn]] - void debugThrow(E && error, const Env & env, const Expr & expr) + void debugThrowLastTrace(E && error) { - if (debugRepl) - runDebugRepl(&error, env, expr); - - throw std::move(error); + debugThrow(error, nullptr, nullptr); } template<class E> [[gnu::noinline, gnu::noreturn]] - void debugThrowLastTrace(E && e) + void debugThrow(E && error, const Env * env, const Expr * expr) { - // Call this in the situation where Expr and Env are inaccessible. - // The debugger will start in the last context that's in the - // DebugTrace stack. - if (debugRepl && !debugTraces.empty()) { - const DebugTrace & last = debugTraces.front(); - runDebugRepl(&e, last.env, last.expr); + if (debugRepl && ((env && expr) || !debugTraces.empty())) { + if (!env || !expr) { + const DebugTrace & last = debugTraces.front(); + env = &last.env; + expr = &last.expr; + } + runDebugRepl(&error, *env, *expr); } - throw std::move(e); + throw std::move(error); } + // This is dangerous, but gets in line with the idea that error creation and + // throwing should not allocate on the stack of hot functions. + // as long as errors are immediately thrown, it works. + ErrorBuilder * errorBuilder; + + template<typename... Args> + [[nodiscard, gnu::noinline]] + ErrorBuilder & error(const Args & ... args) { + errorBuilder = ErrorBuilder::create(*this, args...); + return *errorBuilder; + } private: SrcToStore srcToStore; @@ -283,8 +328,8 @@ public: /* Evaluation the expression, then verify that it has the expected type. */ inline bool evalBool(Env & env, Expr * e); - inline bool evalBool(Env & env, Expr * e, const PosIdx pos); - inline void evalAttrs(Env & env, Expr * e, Value & v); + inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); + inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); /* If `v' is a thunk, enter it and overwrite `v' with the result of the evaluation of the thunk. If `v' is a delayed function @@ -300,89 +345,25 @@ public: void forceValueDeep(Value & v); /* Force `v', and then verify that it has the expected type. */ - NixInt forceInt(Value & v, const PosIdx pos); - NixFloat forceFloat(Value & v, const PosIdx pos); - bool forceBool(Value & v, const PosIdx pos); + NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); + NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); + bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceAttrs(Value & v, const PosIdx pos); + void forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx); template <typename Callable> - inline void forceAttrs(Value & v, Callable getPos); - - inline void forceList(Value & v, const PosIdx pos); - void forceFunction(Value & v, const PosIdx pos); // either lambda or primop - std::string_view forceString(Value & v, const PosIdx pos = noPos); - std::string_view forceString(Value & v, PathSet & context, const PosIdx pos = noPos); - std::string_view forceStringNoCtx(Value & v, const PosIdx pos = noPos); - - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2, const std::string & s3, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2, const std::string & s3, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const char * s, const std::string & s2, const std::string & s3); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const char * s, const std::string & s2, const std::string & s3); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx pos, const Suggestions & suggestions, const char * s, const std::string & s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwEvalError(const PosIdx p1, const char * s, const Symbol sym, const PosIdx p2, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, const Value & v); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, const Value & v, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const char * s, const ExprLambda & fun, const Symbol s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const PosIdx pos, const Suggestions & suggestions, const char * s, const ExprLambda & fun, const Symbol s2, - Env & env, Expr & expr); - [[gnu::noinline, gnu::noreturn]] - void throwTypeError(const char * s, const Value & v, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwAssertionError(const PosIdx pos, const char * s, const std::string & s1, - Env & env, Expr & expr); - - [[gnu::noinline, gnu::noreturn]] - void throwUndefinedVarError(const PosIdx pos, const char * s, const std::string & s1, - Env & env, Expr & expr); + inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); - [[gnu::noinline, gnu::noreturn]] - void throwMissingArgumentError(const PosIdx pos, const char * s, const std::string & s1, - Env & env, Expr & expr); + inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop + std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); [[gnu::noinline]] void addErrorTrace(Error & e, const char * s, const std::string & s2) const; [[gnu::noinline]] - void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2) const; + void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; public: /* Return true iff the value `v' denotes a derivation (i.e. a @@ -397,18 +378,19 @@ public: booleans and lists to a string. If `copyToStore' is set, referenced paths are copied to the Nix store as a side effect. */ BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, + std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - std::string copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(PathSet & context, const Path & path); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute path. Nothing is copied to the store. */ - Path coerceToPath(const PosIdx pos, Value & v, PathSet & context); + Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); /* Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context); + StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); public: @@ -457,14 +439,18 @@ private: friend struct ExprAttrs; friend struct ExprLet; - Expr * parse(char * text, size_t length, FileOrigin origin, const PathView path, - const PathView basePath, std::shared_ptr<StaticEnv> & staticEnv); + Expr * parse( + char * text, + size_t length, + Pos::Origin origin, + Path basePath, + std::shared_ptr<StaticEnv> & staticEnv); public: /* Do a deep equality test between two values. That is, list elements and attributes are compared recursively. */ - bool eqValues(Value & v1, Value & v2); + bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); @@ -499,7 +485,7 @@ public: void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, PosIdx pos); - void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /* Print statistics. */ void printStats(); @@ -568,10 +554,6 @@ struct DebugTraceStacker { std::string_view showType(ValueType type); std::string showType(const Value & v); -/* Decode a context string ‘!<name>!<path>’ into a pair <path, - name>. */ -NixStringContextElem decodeContext(const Store & store, std::string_view s); - /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); @@ -588,14 +570,25 @@ struct EvalSettings : Config { EvalSettings(); - static Strings getDefaultNixPath(); + Strings getDefaultNixPath(); + + static bool isPseudoUrl(std::string_view s); + + static std::string resolvePseudoUrl(std::string_view url); Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "Whether builtin functions that allow executing native code should be enabled."}; Setting<Strings> nixPath{ - this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for `<...>` file references."}; + this, {}, "nix-path", + R"( + List of directories to be searched for `<...>` file references. + + If [pure evaluation](#conf-pure-eval) is disabled, + this is initialised using the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable, or, if it is unset and [restricted evaluation](#conf-restrict-eval) + is disabled, a default search path including the user's and `root`'s channels. + )"}; Setting<bool> restrictEval{ this, false, "restrict-eval", @@ -662,6 +655,13 @@ extern EvalSettings evalSettings; static const std::string corepkgsPrefix{"/__corepkgs__/"}; +template<class ErrorType> +void ErrorBuilder::debugThrow() +{ + // NOTE: We always use the -LastTrace version as we push the new trace in withFrame() + state.debugThrowLastTrace(ErrorType(info)); +} + } #include "eval-inline.hh" diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 6df95f1f0..89ddbde7e 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -56,7 +56,7 @@ void ConfigFile::apply() auto tlname = get(trustedList, name); if (auto saved = tlname ? get(*tlname, valueS) : nullptr) { trusted = *saved; - warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); + printInfo("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name, valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 119c556ac..336eb274d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -143,7 +143,7 @@ static FlakeInput parseFlakeInput(EvalState & state, } catch (Error & e) { e.addTrace( state.positions[attr.pos], - hintfmt("in flake attribute '%s'", state.symbols[attr.name])); + hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); throw; } } @@ -152,7 +152,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { input.ref = FlakeRef::fromAttrs(attrs); } catch (Error & e) { - e.addTrace(state.positions[pos], hintfmt("in flake input")); + e.addTrace(state.positions[pos], hintfmt("while evaluating flake input")); throw; } else { @@ -220,7 +220,7 @@ static Flake getFlake( Value vInfo; state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({flakeFile, foFile}, 0, 0)); + expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); @@ -259,28 +259,28 @@ static Flake getFlake( if (setting.value->type() == nString) flake.config.settings.emplace( state.symbols[setting.name], - std::string(state.forceStringNoCtx(*setting.value, setting.pos))); + std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { PathSet emptyContext = {}; flake.config.settings.emplace( state.symbols[setting.name], - state.coerceToString(setting.pos, *setting.value, emptyContext, false, true, true) .toOwned()); + state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); } else if (setting.value->type() == nInt) flake.config.settings.emplace( state.symbols[setting.name], - state.forceInt(*setting.value, setting.pos)); + state.forceInt(*setting.value, setting.pos, "")); else if (setting.value->type() == nBool) flake.config.settings.emplace( state.symbols[setting.name], - Explicit<bool> { state.forceBool(*setting.value, setting.pos) }); + Explicit<bool> { state.forceBool(*setting.value, setting.pos, "") }); else if (setting.value->type() == nList) { std::vector<std::string> ss; for (auto elem : setting.value->listItems()) { if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", state.symbols[setting.name], showType(*setting.value)); - ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos)); + ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, "")); } flake.config.settings.emplace(state.symbols[setting.name], ss); } @@ -353,7 +353,7 @@ LockedFlake lockFlake( std::function<void( const FlakeInputs & flakeInputs, - std::shared_ptr<Node> node, + ref<Node> node, const InputPath & inputPathPrefix, std::shared_ptr<const Node> oldNode, const InputPath & lockRootPath, @@ -362,9 +362,15 @@ LockedFlake lockFlake( computeLocks; computeLocks = [&]( + /* The inputs of this node, either from flake.nix or + flake.lock. */ const FlakeInputs & flakeInputs, - std::shared_ptr<Node> node, + /* The node whose locks are to be updated.*/ + ref<Node> node, + /* The path to this node in the lock file graph. */ const InputPath & inputPathPrefix, + /* The old node, if any, from which locks can be + copied. */ std::shared_ptr<const Node> oldNode, const InputPath & lockRootPath, const Path & parentPath, @@ -452,7 +458,7 @@ LockedFlake lockFlake( /* Copy the input from the old lock since its flakeref didn't change and there is no override from a higher level flake. */ - auto childNode = std::make_shared<LockedNode>( + auto childNode = make_ref<LockedNode>( oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); @@ -481,7 +487,7 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { - if (! trustLock) { + if (!trustLock) { // It is possible that the flake has changed, // so we must confirm all the follows that are in the lock file are also in the flake. auto overridePath(inputPath); @@ -521,8 +527,8 @@ LockedFlake lockFlake( this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowMutable && !input.ref->input.isLocked()) - throw Error("cannot update flake input '%s' in pure mode", inputPathS); + if (!lockFlags.allowUnlocked && !input.ref->input.isLocked()) + throw Error("cannot update unlocked flake input '%s' in pure mode", inputPathS); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -544,7 +550,7 @@ LockedFlake lockFlake( auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); - auto childNode = std::make_shared<LockedNode>(inputFlake.lockedRef, ref); + auto childNode = make_ref<LockedNode>(inputFlake.lockedRef, ref); node->inputs.insert_or_assign(id, childNode); @@ -564,15 +570,19 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast<const Node>(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - oldLock ? lockRootPath : inputPath, localPath, false); + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + oldLock ? lockRootPath : inputPath, + localPath, + false); } else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); - node->inputs.insert_or_assign(id, - std::make_shared<LockedNode>(lockedRef, ref, false)); + + auto childNode = make_ref<LockedNode>(lockedRef, ref, false); + + node->inputs.insert_or_assign(id, childNode); } } @@ -587,8 +597,13 @@ LockedFlake lockFlake( auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); computeLocks( - flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false); + flake.inputs, + newLockFile.root, + {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root.get_ptr(), + {}, + parentPath, + false); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -611,9 +626,9 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (auto sourcePath = topRef.input.getSourcePath()) { - if (!newLockFile.isImmutable()) { + if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) - warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); @@ -726,7 +741,7 @@ void callFlake(EvalState & state, static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos)); + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); @@ -737,7 +752,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V .updateLockFile = false, .writeLockFile = false, .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowMutable = !evalSettings.pureEval, + .allowUnlocked = !evalSettings.pureEval, }), v); } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 524b18af1..10301d8aa 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -108,11 +108,11 @@ struct LockFlags bool applyNixConfig = false; - /* Whether mutable flake references (i.e. those without a Git + /* Whether unlocked flake references (i.e. those without a Git revision or similar) without a corresponding lock are - allowed. Mutable flake references with a lock are always + allowed. Unlocked flake references with a lock are always allowed. */ - bool allowMutable = true; + bool allowUnlocked = true; /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index eede493f8..08adbe0c9 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -238,15 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } -std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec( +std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec( const std::string & url, const std::optional<Path> & baseDir, bool allowMissing, bool isFlake) { - auto [prefix, outputsSpec] = parseOutputsSpec(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake); - return {std::move(flakeRef), fragment, outputsSpec}; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + return {std::move(flakeRef), fragment, extendedOutputsSpec}; } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index fe4f67193..c4142fc20 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "hash.hh" #include "fetchers.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include <variant> @@ -35,7 +35,7 @@ typedef std::string FlakeId; struct FlakeRef { - /* fetcher-specific representation of the input, sufficient to + /* Fetcher-specific representation of the input, sufficient to perform the fetch operation. */ fetchers::Input input; @@ -80,7 +80,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( const std::string & url, const std::optional<Path> & baseDir = {}); -std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec( +std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec( const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false, diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 629d2e669..a74e68c9c 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -31,7 +31,7 @@ FlakeRef getFlakeRef( } LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) + : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { @@ -49,15 +49,15 @@ std::shared_ptr<Node> LockFile::findInput(const InputPath & path) { auto pos = root; - if (!pos) return {}; - for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - pos = findInput(*follows); - if (!pos) return {}; + if (auto p = findInput(*follows)) + pos = ref(p); + else + return {}; } } else return {}; @@ -72,7 +72,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); - std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap; + std::map<std::string, ref<Node>> nodeMap; std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs; @@ -93,12 +93,12 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = std::make_shared<LockedNode>(*jsonNode2); + auto input = make_ref<LockedNode>(*jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } - if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second)) - node.inputs.insert_or_assign(i.key(), child); + if (auto child = k->second.dynamic_pointer_cast<LockedNode>()) + node.inputs.insert_or_assign(i.key(), ref(child)); else // FIXME: replace by follows node throw Error("lock file contains cycle to root node"); @@ -122,9 +122,9 @@ nlohmann::json LockFile::toJSON() const std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys; std::unordered_set<std::string> keys; - std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode; + std::function<std::string(const std::string & key, ref<const Node> node)> dumpNode; - dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string + dumpNode = [&](std::string key, ref<const Node> node) -> std::string { auto k = nodeKeys.find(node); if (k != nodeKeys.end()) @@ -159,10 +159,11 @@ nlohmann::json LockFile::toJSON() const n["inputs"] = std::move(inputs); } - if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) { + if (auto lockedNode = node.dynamic_pointer_cast<const LockedNode>()) { n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); - if (!lockedNode->isFlake) n["flake"] = false; + if (!lockedNode->isFlake) + n["flake"] = false; } nodes[key] = std::move(n); @@ -201,13 +202,13 @@ void LockFile::write(const Path & path) const writeFile(path, fmt("%s\n", *this)); } -bool LockFile::isImmutable() const +std::optional<FlakeRef> LockFile::isUnlocked() const { - std::unordered_set<std::shared_ptr<const Node>> nodes; + std::set<ref<const Node>> nodes; - std::function<void(std::shared_ptr<const Node> node)> visit; + std::function<void(ref<const Node> node)> visit; - visit = [&](std::shared_ptr<const Node> node) + visit = [&](ref<const Node> node) { if (!nodes.insert(node).second) return; for (auto & i : node->inputs) @@ -218,12 +219,13 @@ bool LockFile::isImmutable() const visit(root); for (auto & i : nodes) { - if (i == root) continue; - auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); - if (lockedNode && !lockedNode->lockedRef.input.isLocked()) return false; + if (i == ref<const Node>(root)) continue; + auto node = i.dynamic_pointer_cast<const LockedNode>(); + if (node && !node->lockedRef.input.isLocked()) + return node->lockedRef; } - return true; + return {}; } bool LockFile::operator ==(const LockFile & other) const @@ -247,12 +249,12 @@ InputPath parseInputPath(std::string_view s) std::map<InputPath, Node::Edge> LockFile::getAllInputs() const { - std::unordered_set<std::shared_ptr<Node>> done; + std::set<ref<Node>> done; std::map<InputPath, Node::Edge> res; - std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse; + std::function<void(const InputPath & prefix, ref<Node> node)> recurse; - recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node) + recurse = [&](const InputPath & prefix, ref<Node> node) { if (!done.insert(node).second) return; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 96f1edc76..02e9bdfbc 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -20,7 +20,7 @@ struct LockedNode; type LockedNode. */ struct Node : std::enable_shared_from_this<Node> { - typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge; + typedef std::variant<ref<LockedNode>, InputPath> Edge; std::map<FlakeId, Edge> inputs; @@ -47,11 +47,13 @@ struct LockedNode : Node struct LockFile { - std::shared_ptr<Node> root = std::make_shared<Node>(); + ref<Node> root = make_ref<Node>(); LockFile() {}; LockFile(const nlohmann::json & json, const Path & path); + typedef std::map<ref<const Node>, std::string> KeyMap; + nlohmann::json toJSON() const; std::string to_string() const; @@ -60,7 +62,8 @@ struct LockFile void write(const Path & path) const; - bool isImmutable() const; + /* Check whether this lock file has any unlocked inputs. */ + std::optional<FlakeRef> isUnlocked() const; bool operator ==(const LockFile & other) const; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 346741dd5..1602fbffb 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -51,7 +51,7 @@ std::string DrvInfo::queryName() const if (name == "" && attrs) { auto i = attrs->find(state->sName); if (i == attrs->end()) throw TypeError("derivation name missing"); - name = state->forceStringNoCtx(*i->value); + name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation"); } return name; } @@ -61,7 +61,7 @@ std::string DrvInfo::querySystem() const { if (system == "" && attrs) { auto i = attrs->find(state->sSystem); - system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos); + system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation"); } return system; } @@ -75,7 +75,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const if (i == attrs->end()) drvPath = {std::nullopt}; else - drvPath = {state->coerceToStorePath(i->pos, *i->value, context)}; + drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")}; } return drvPath.value_or(std::nullopt); } @@ -95,7 +95,7 @@ StorePath DrvInfo::queryOutPath() const Bindings::iterator i = attrs->find(state->sOutPath); PathSet context; if (i != attrs->end()) - outPath = state->coerceToStorePath(i->pos, *i->value, context); + outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } if (!outPath) throw UnimplementedError("CA derivations are not yet supported"); @@ -109,23 +109,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* Get the ‘outputs’ list. */ Bindings::iterator i; if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value, i->pos); + state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation"); /* For each output... */ for (auto elem : i->value->listItems()) { - std::string output(state->forceStringNoCtx(*elem, i->pos)); + std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation")); if (withPaths) { /* Evaluate the corresponding set. */ Bindings::iterator out = attrs->find(state->symbols.create(output)); if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value, i->pos); + state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation"); /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? PathSet context; - outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context)); + outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); } @@ -137,7 +137,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall return outputs; Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos)) { + if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { Outputs result; auto out = outputs.find(queryOutputName()); if (out == outputs.end()) @@ -150,7 +150,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ const Value * outTI = queryMeta("outputsToInstall"); if (!outTI) return outputs; - const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; @@ -169,7 +169,7 @@ std::string DrvInfo::queryOutputName() const { if (outputName == "" && attrs) { Bindings::iterator i = attrs->find(state->sOutputName); - outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : ""; } return outputName; } @@ -181,7 +181,7 @@ Bindings * DrvInfo::getMeta() if (!attrs) return 0; Bindings::iterator a = attrs->find(state->sMeta); if (a == attrs->end()) return 0; - state->forceAttrs(*a->value, a->pos); + state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation"); meta = a->value->attrs; return meta; } @@ -382,7 +382,7 @@ static void getDerivations(EvalState & state, Value & vIn, `recurseForDerivations = true' attribute. */ if (i->value->type() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); - if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos)) + if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 016631647..2171e769b 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,6 +6,7 @@ libexpr_DIR := $(d) libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ @@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/value/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 80f7a492b..60ffb5dba 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 7c623a07d..eb6f062b4 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -8,6 +8,58 @@ namespace nix { +struct PosAdapter : AbstractPos +{ + Pos::Origin origin; + + PosAdapter(Pos::Origin origin) + : origin(std::move(origin)) + { + } + + std::optional<std::string> getSource() const override + { + return std::visit(overloaded { + [](const Pos::none_tag &) -> std::optional<std::string> { + return std::nullopt; + }, + [](const Pos::Stdin & s) -> std::optional<std::string> { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const Pos::String & s) -> std::optional<std::string> { + // Get rid of the null terminators added by the parser. + return std::string(s.source->c_str()); + }, + [](const Path & path) -> std::optional<std::string> { + try { + return readFile(path); + } catch (Error &) { + return std::nullopt; + } + } + }, origin); + } + + void print(std::ostream & out) const override + { + std::visit(overloaded { + [&](const Pos::none_tag &) { out << "«none»"; }, + [&](const Pos::Stdin &) { out << "«stdin»"; }, + [&](const Pos::String & s) { out << "«string»"; }, + [&](const Path & path) { out << path; } + }, origin); + } +}; + +Pos::operator std::shared_ptr<AbstractPos>() const +{ + auto pos = std::make_shared<PosAdapter>(origin); + pos->line = line; + pos->column = column; + return pos; +} + /* Displaying abstract syntax trees. */ static void showString(std::ostream & str, std::string_view s) @@ -248,24 +300,10 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const std::ostream & operator << (std::ostream & str, const Pos & pos) { - if (!pos) + if (auto pos2 = (std::shared_ptr<AbstractPos>) pos) { + str << *pos2; + } else str << "undefined position"; - else - { - auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%"); - switch (pos.origin) { - case foFile: - f % (const std::string &) pos.file; - break; - case foStdin: - case foString: - f % "(string)"; - break; - default: - throw Error("unhandled Pos origin!"); - } - str << (f % pos.line % pos.column).str(); - } return str; } @@ -289,7 +327,6 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) } - /* Computing levels/displacements for variables. */ void Expr::bindVars(EvalState & es, const std::shared_ptr<const StaticEnv> & env) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5eb022770..4a81eaa47 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -8,7 +8,6 @@ #include "error.hh" #include "chunked-vector.hh" - namespace nix { @@ -23,15 +22,22 @@ MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); /* Position objects. */ - struct Pos { - std::string file; - FileOrigin origin; uint32_t line; uint32_t column; + struct none_tag { }; + struct Stdin { ref<std::string> source; }; + struct String { ref<std::string> source; }; + + typedef std::variant<none_tag, Stdin, String, Path> Origin; + + Origin origin; + explicit operator bool() const { return line > 0; } + + operator std::shared_ptr<AbstractPos>() const; }; class PosIdx { @@ -47,7 +53,11 @@ public: explicit operator bool() const { return id > 0; } - bool operator<(const PosIdx other) const { return id < other.id; } + bool operator <(const PosIdx other) const { return id < other.id; } + + bool operator ==(const PosIdx other) const { return id == other.id; } + + bool operator !=(const PosIdx other) const { return id != other.id; } }; class PosTable @@ -61,13 +71,13 @@ public: // current origins.back() can be reused or not. mutable uint32_t idx = std::numeric_limits<uint32_t>::max(); - explicit Origin(uint32_t idx): idx(idx), file{}, origin{} {} + // Used for searching in PosTable::[]. + explicit Origin(uint32_t idx): idx(idx), origin{Pos::none_tag()} {} public: - const std::string file; - const FileOrigin origin; + const Pos::Origin origin; - Origin(std::string file, FileOrigin origin): file(std::move(file)), origin(origin) {} + Origin(Pos::Origin origin): origin(origin) {} }; struct Offset { @@ -107,7 +117,7 @@ public: [] (const auto & a, const auto & b) { return a.idx < b.idx; }); const auto origin = *std::prev(pastOrigin); const auto offset = offsets[idx]; - return {origin.file, origin.origin, offset.line, offset.column}; + return {offset.line, offset.column, origin.origin}; } }; @@ -176,7 +186,7 @@ struct ExprString : Expr { std::string s; Value v; - ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; + ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -223,7 +233,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; + ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -233,7 +243,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 7c9b5a2db..dec5818fc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -34,11 +34,6 @@ namespace nix { Path basePath; PosTable::Origin origin; std::optional<ErrorInfo> error; - ParseData(EvalState & state, PosTable::Origin origin) - : state(state) - , symbols(state.symbols) - , origin(std::move(origin)) - { }; }; struct ParserFormals { @@ -95,7 +90,7 @@ static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, cons } -static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, +static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos, const nix::EvalState & state) { AttrPath::iterator i; @@ -193,7 +188,7 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals, static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, - std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> & es) + std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es) { if (es.empty()) return new ExprString(""); @@ -273,7 +268,7 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(s2)); + es2->emplace_back(i->first, new ExprString(std::move(s2))); }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); @@ -405,21 +400,21 @@ expr_op | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } - | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } - | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } - | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } - | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } - | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '<' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } + | expr_op AND expr_op { $$ = new ExprOpAnd(makeCurPos(@2, data), $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(CUR_POS, false, new std::vector<std::pair<PosIdx, Expr *>>({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } - | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } - | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } - | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } - | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } + { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector<std::pair<PosIdx, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__div")), {$1, $3}); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(makeCurPos(@2, data), $1, $3); } | expr_app ; @@ -436,14 +431,14 @@ expr_app expr_select : expr_simple '.' attrpath - { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; } | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } - | expr_simple { $$ = $1; } + | expr_simple ; expr_simple @@ -458,9 +453,10 @@ expr_simple | FLOAT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, *$2); + $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2)); + delete $2; } - | path_start PATH_END { $$ = $1; } + | path_start PATH_END | path_start string_parts_interpolated PATH_END { $2->insert($2->begin(), {makeCurPos(@1, data), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); @@ -470,7 +466,7 @@ expr_simple $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__findFile")), {new ExprVar(data->symbols.create("__nixPath")), - new ExprString(path)}); + new ExprString(std::move(path))}); } | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); @@ -538,7 +534,7 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data), data->state); } + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { @@ -547,6 +543,7 @@ binds auto pos = makeCurPos(@3, data); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } + delete $3; } | binds INHERIT '(' expr ')' attrs ';' { $$ = $1; @@ -556,6 +553,7 @@ binds dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } + delete $6; } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; @@ -601,7 +599,7 @@ attrpath ; attr - : ID { $$ = $1; } + : ID | OR_KW { $$ = {"or", 2}; } ; @@ -617,9 +615,9 @@ expr_list formals : formal ',' formals - { $$ = $3; $$->formals.push_back(*$1); } + { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } | formal - { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; } + { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } | { $$ = new ParserFormals; $$->ellipsis = false; } | ELLIPSIS @@ -643,29 +641,26 @@ formal #include "filetransfer.hh" #include "fetchers.hh" #include "store-api.hh" +#include "flake/flake.hh" namespace nix { -Expr * EvalState::parse(char * text, size_t length, FileOrigin origin, - const PathView path, const PathView basePath, std::shared_ptr<StaticEnv> & staticEnv) +Expr * EvalState::parse( + char * text, + size_t length, + Pos::Origin origin, + Path basePath, + std::shared_ptr<StaticEnv> & staticEnv) { yyscan_t scanner; - std::string file; - switch (origin) { - case foFile: - file = path; - break; - case foStdin: - case foString: - file = text; - break; - default: - assert(false); - } - ParseData data(*this, {file, origin}); - data.basePath = basePath; + ParseData data { + .state = *this, + .symbols = symbols, + .basePath = std::move(basePath), + .origin = {origin}, + }; yylex_init(&scanner); yy_scan_buffer(text, length, scanner); @@ -717,14 +712,15 @@ Expr * EvalState::parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv auto buffer = readFile(path); // readFile should have left some extra space for terminators buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv); + return parse(buffer.data(), buffer.size(), path, dirOf(path), staticEnv); } -Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv) +Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv) { - s.append("\0\0", 2); - return parse(s.data(), s.size(), foString, "", basePath, staticEnv); + auto s = make_ref<std::string>(std::move(s_)); + s->append("\0\0", 2); + return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); } @@ -740,7 +736,8 @@ Expr * EvalState::parseStdin() auto buffer = drainFD(0); // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv); + auto s = make_ref<std::string>(std::move(buffer)); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, absPath("."), staticBaseEnv); } @@ -788,13 +785,13 @@ Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, c if (hasPrefix(path, "nix/")) return concatStrings(corepkgsPrefix, path.substr(4)); - debugThrowLastTrace(ThrownError({ + debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), .errPos = positions[pos] - })); + }), 0, 0); } @@ -805,17 +802,28 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl std::pair<bool, std::string> res; - if (isUri(elem.second)) { + if (EvalSettings::isPseudoUrl(elem.second)) { try { - res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).first.storePath) }; + auto storePath = fetchers::downloadTarball( + store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath; + res = { true, store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) }); res = { false, "" }; } - } else { + } + + else if (hasPrefix(elem.second, "flake:")) { + settings.requireExperimentalFeature(Xp::Flakes); + auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", elem.second); + auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; + res = { true, store->toRealPath(storePath) }; + } + + else { auto path = absPath(elem.second); if (pathExists(path)) res = { true, path }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8a4c19f7c..fb7fc3ddb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context) std::vector<DerivedPath::Built> drvs; StringMap res; - for (auto & i : context) { - auto [ctx, outputName] = decodeContext(*store, i); - auto ctxS = store->printStorePath(ctx); - if (!store->isValidPath(ctx)) - debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx))); - if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back({ctx, {outputName}}); - } else { - res.insert_or_assign(ctxS, ctxS); - } + for (auto & c_ : context) { + auto ensureValid = [&](const StorePath & p) { + if (!store->isValidPath(p)) + debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); + }; + auto c = NixStringContextElem::parse(*store, c_); + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + drvs.push_back(DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }); + ensureValid(b.drvPath); + }, + [&](const NixStringContextElem::Opaque & o) { + auto ctxS = store->printStorePath(o.path); + res.insert_or_assign(ctxS, ctxS); + ensureValid(o.path); + }, + [&](const NixStringContextElem::DrvDeep & d) { + /* Treat same as Opaque */ + auto ctxS = store->printStorePath(d.drvPath); + res.insert_or_assign(ctxS, ctxS); + ensureValid(d.drvPath); + }, + }, c.raw()); } if (drvs.empty()) return {}; @@ -68,16 +84,12 @@ StringMap EvalState::realiseContext(const PathSet & context) store->buildPaths(buildReqs); /* Get all the output paths corresponding to the placeholders we had */ - for (auto & [drvPath, outputs] : drvs) { - const auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - auto outputPath = get(outputPaths, outputName); - if (!outputPath) - debugThrowLastTrace(Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName)); + for (auto & drv : drvs) { + auto outputs = resolveDerivedPath(*store, drv); + for (auto & [outputName, outputPath] : outputs) { res.insert_or_assign( - downstreamPlaceholder(*store, drvPath, outputName), - store->printStorePath(*outputPath) + downstreamPlaceholder(*store, drv.drvPath, outputName), + store->printStorePath(outputPath) ); } } @@ -102,15 +114,7 @@ static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const Re { PathSet context; - auto path = [&]() - { - try { - return state.coerceToPath(pos, v, context); - } catch (Error & e) { - e.addTrace(state.positions[pos], "while realising the context of a path"); - throw; - } - }(); + auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { StringMap rewrites = state.realiseContext(context); @@ -197,9 +201,9 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v , "/"), **state.vImportedDrvToDerivation); } - state.forceFunction(**state.vImportedDrvToDerivation, pos); + state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); v.mkApp(*state.vImportedDrvToDerivation, w); - state.forceAttrs(v, pos); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } else if (path == corepkgsPrefix + "fetchurl.nix") { @@ -212,7 +216,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v if (!vScope) state.evalFile(path, v); else { - state.forceAttrs(*vScope, pos); + state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; @@ -248,6 +252,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { static RegisterPrimOp primop_import({ .name = "import", .args = {"path"}, + // TODO turn "normal path values" into link below .doc = R"( Load, parse and return the Nix expression in the file *path*. If *path* is a directory, the file ` default.nix ` in that directory @@ -261,7 +266,7 @@ static RegisterPrimOp primop_import({ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*) - > are [normal path values](language-values.md). + > are normal [path values](@docroot@/language/values.md#type-path). A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression @@ -316,7 +321,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu { auto path = realisePath(state, pos, *args[0]); - std::string sym(state.forceStringNoCtx(*args[1], pos)); + std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) @@ -341,48 +346,44 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu /* Execute a program and parse its output */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.exec"); auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("at least one argument to 'exec' required"), - .errPos = state.positions[pos] - })); + state.error("at least one argument to 'exec' required").atPos(pos).debugThrow<EvalError>(); PathSet context; - auto program = state.coerceToString(pos, *elems[0], context, false, false).toOwned(); + auto program = state.coerceToString(pos, *elems[0], context, + "while evaluating the first element of the argument passed to builtins.exec", + false, false).toOwned(); Strings commandArgs; for (unsigned int i = 1; i < args[0]->listSize(); ++i) { - commandArgs.push_back(state.coerceToString(pos, *elems[i], context, false, false).toOwned()); + commandArgs.push_back( + state.coerceToString(pos, *elems[i], context, + "while evaluating an element of the argument passed to builtins.exec", + false, false).toOwned()); } try { auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", - program, e.path), - .errPos = state.positions[pos] - })); + state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow<EvalError>(); } auto output = runProgram(program, true, commandArgs); Expr * parsed; try { - auto base = state.positions[pos]; - parsed = state.parseExprFromString(std::move(output), base.file); + parsed = state.parseExprFromString(std::move(output), "/"); } catch (Error & e) { - e.addTrace(state.positions[pos], "While parsing the output from '%1%'", program); + e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program); throw; } try { state.eval(parsed, v); } catch (Error & e) { - e.addTrace(state.positions[pos], "While evaluating the output from '%1%'", program); + e.addTrace(state.positions[pos], "while evaluating the output from '%1%'", program); throw; } } - /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { @@ -533,42 +534,69 @@ static RegisterPrimOp primop_isPath({ .fun = prim_isPath, }); +template<typename Callable> + static inline void withExceptionContext(Trace trace, Callable&& func) +{ + try + { + func(); + } + catch(Error & e) + { + e.pushTrace(trace); + throw; + } +} + struct CompareValues { EvalState & state; + const PosIdx pos; + const std::string_view errorCtx; - CompareValues(EvalState & state) : state(state) { }; + CompareValues(EvalState & state, const PosIdx pos, const std::string_view && errorCtx) : state(state), pos(pos), errorCtx(errorCtx) { }; bool operator () (Value * v1, Value * v2) const { - if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint < v2->integer; - if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer < v2->fpoint; - if (v1->type() != v2->type()) - state.debugThrowLastTrace(EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2))); - switch (v1->type()) { - case nInt: - return v1->integer < v2->integer; - case nFloat: - return v1->fpoint < v2->fpoint; - case nString: - return strcmp(v1->string.s, v2->string.s) < 0; - case nPath: - return strcmp(v1->path, v2->path) < 0; - case nList: - // Lexicographic comparison - for (size_t i = 0;; i++) { - if (i == v2->listSize()) { - return false; - } else if (i == v1->listSize()) { - return true; - } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { - return (*this)(v1->listElems()[i], v2->listElems()[i]); + return (*this)(v1, v2, errorCtx); + } + + bool operator () (Value * v1, Value * v2, std::string_view errorCtx) const + { + try { + if (v1->type() == nFloat && v2->type() == nInt) + return v1->fpoint < v2->integer; + if (v1->type() == nInt && v2->type() == nFloat) + return v1->integer < v2->fpoint; + if (v1->type() != v2->type()) + state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow<EvalError>(); + switch (v1->type()) { + case nInt: + return v1->integer < v2->integer; + case nFloat: + return v1->fpoint < v2->fpoint; + case nString: + return strcmp(v1->string.s, v2->string.s) < 0; + case nPath: + return strcmp(v1->path, v2->path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i], pos, errorCtx)) { + return (*this)(v1->listElems()[i], v2->listElems()[i], "while comparing two list elements"); + } } - } - default: - state.debugThrowLastTrace(EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2))); + default: + state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow<EvalError>(); + } + } catch (Error & e) { + if (!errorCtx.empty()) + e.addTrace(nullptr, errorCtx); + throw; } } }; @@ -583,105 +611,67 @@ typedef std::list<Value *> ValueList; static Bindings::iterator getAttr( EvalState & state, - std::string_view funcName, Symbol attrSym, Bindings * attrSet, - const PosIdx pos) + std::string_view errorCtx) { Bindings::iterator value = attrSet->find(attrSym); if (value == attrSet->end()) { - hintformat errorMsg = hintfmt( - "attribute '%s' missing for call to '%s'", - state.symbols[attrSym], - funcName - ); - - auto aPos = attrSet->pos; - if (!aPos) { - state.debugThrowLastTrace(TypeError({ - .msg = errorMsg, - .errPos = state.positions[pos], - })); - } else { - auto e = TypeError({ - .msg = errorMsg, - .errPos = state.positions[aPos], - }); - - // Adding another trace for the function name to make it clear - // which call received wrong arguments. - e.addTrace(state.positions[pos], hintfmt("while invoking '%s'", funcName)); - state.debugThrowLastTrace(e); - } + state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow<TypeError>(); } - return value; } static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], noPos, "while evaluating the first argument passed to builtins.genericClosure"); /* Get the start set. */ - Bindings::iterator startSet = getAttr( - state, - "genericClosure", - state.sStartSet, - args[0]->attrs, - pos - ); + Bindings::iterator startSet = getAttr(state, state.sStartSet, args[0]->attrs, "in the attrset passed as argument to builtins.genericClosure"); - state.forceList(*startSet->value, pos); + state.forceList(*startSet->value, noPos, "while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure"); ValueList workSet; for (auto elem : startSet->value->listItems()) workSet.push_back(elem); - /* Get the operator. */ - Bindings::iterator op = getAttr( - state, - "genericClosure", - state.sOperator, - args[0]->attrs, - pos - ); + if (startSet->value->listSize() == 0) { + v = *startSet->value; + return; + } - state.forceValue(*op->value, pos); + /* Get the operator. */ + Bindings::iterator op = getAttr(state, state.sOperator, args[0]->attrs, "in the attrset passed as argument to builtins.genericClosure"); + state.forceFunction(*op->value, noPos, "while evaluating the 'operator' attribute passed as argument to builtins.genericClosure"); - /* Construct the closure by applying the operator to element of + /* Construct the closure by applying the operator to elements of `workSet', adding the result to `workSet', continuing until no new elements are found. */ ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - auto cmp = CompareValues(state); + auto cmp = CompareValues(state, noPos, "while comparing the `key` attributes of two genericClosure elements"); std::set<Value *, decltype(cmp)> doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); - state.forceAttrs(*e, pos); + state.forceAttrs(*e, noPos, "while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"); - Bindings::iterator key = - e->attrs->find(state.sKey); - if (key == e->attrs->end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'key' required"), - .errPos = state.positions[pos] - })); - state.forceValue(*key->value, pos); + Bindings::iterator key = getAttr(state, state.sKey, e->attrs, "in one of the attrsets generated by (or initially passed to) builtins.genericClosure"); + state.forceValue(*key->value, noPos); if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ - Value call; - call.mkApp(op->value, e); - state.forceList(call, pos); + Value newElements; + state.callFunction(*op->value, 1, &e, newElements, noPos); + state.forceList(newElements, noPos, "while evaluating the return value of the `operator` passed to builtins.genericClosure"); /* Add the values returned by the operator to the work set. */ - for (auto elem : call.listItems()) { - state.forceValue(*elem, pos); + for (auto elem : newElements.listItems()) { + state.forceValue(*elem, noPos); // "while evaluating one one of the elements returned by the `operator` passed to builtins.genericClosure"); workSet.push_back(elem); } } @@ -749,7 +739,7 @@ static RegisterPrimOp primop_break({ throw Error(ErrorInfo{ .level = lvlInfo, .msg = hintfmt("quit the debugger"), - .errPos = state.positions[noPos], + .errPos = nullptr, }); } } @@ -768,7 +758,8 @@ static RegisterPrimOp primop_abort({ .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto s = state.coerceToString(pos, *args[0], context).toOwned(); + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtins.abort").toOwned(); state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); } }); @@ -786,7 +777,8 @@ static RegisterPrimOp primop_throw({ .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto s = state.coerceToString(pos, *args[0], context).toOwned(); + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtin.throw").toOwned(); state.debugThrowLastTrace(ThrownError(s)); } }); @@ -798,7 +790,10 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * v = *args[1]; } catch (Error & e) { PathSet context; - e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context).toOwned()); + auto message = state.coerceToString(pos, *args[0], context, + "while evaluating the error message passed to builtins.addErrorContext", + false, false).toOwned(); + e.addTrace(nullptr, message, true); throw; } } @@ -811,7 +806,8 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { static void prim_ceil(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), + "while evaluating the first argument passed to builtins.ceil"); v.mkInt(ceil(value)); } @@ -830,7 +826,7 @@ static RegisterPrimOp primop_ceil({ static void prim_floor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos), "while evaluating the first argument passed to builtins.floor"); v.mkInt(floor(value)); } @@ -904,7 +900,7 @@ static RegisterPrimOp primop_tryEval({ /* Return an environment variable. Use with care. */ static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - std::string name(state.forceStringNoCtx(*args[0], pos)); + std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } @@ -1001,6 +997,7 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val * Derivations *************************************************************/ +static void derivationStrictInternal(EvalState & state, const std::string & name, Bindings * attrs, Value & v); /* Construct (as a unobservable side effect) a Nix derivation expression that performs the derivation described by the argument @@ -1011,38 +1008,68 @@ static void prim_second(EvalState & state, const PosIdx pos, Value * * args, Val derivation. */ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - using nlohmann::json; - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.derivationStrict"); + + Bindings * attrs = args[0]->attrs; /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = getAttr( - state, - "derivationStrict", - state.sName, - args[0]->attrs, - pos - ); + Bindings::iterator nameAttr = getAttr(state, state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict"); std::string drvName; - const auto posDrvName = attr->pos; try { - drvName = state.forceStringNoCtx(*attr->value, pos); + drvName = state.forceStringNoCtx(*nameAttr->value, pos, "while evaluating the `name` attribute passed to builtins.derivationStrict"); + } catch (Error & e) { + e.addTrace(state.positions[nameAttr->pos], "while evaluating the derivation attribute 'name'"); + throw; + } + + try { + derivationStrictInternal(state, drvName, attrs, v); } catch (Error & e) { - e.addTrace(state.positions[posDrvName], "while evaluating the derivation attribute 'name'"); + Pos pos = state.positions[nameAttr->pos]; + /* + * Here we make two abuses of the error system + * + * 1. We print the location as a string to avoid a code snippet being + * printed. While the location of the name attribute is a good hint, the + * exact code there is irrelevant. + * + * 2. We mark this trace as a frame trace, meaning that we stop printing + * less important traces from now on. In particular, this prevents the + * display of the automatic "while calling builtins.derivationStrict" + * trace, which is of little use for the public we target here. + * + * Please keep in mind that error reporting is done on a best-effort + * basis in nix. There is no accurate location for a derivation, as it + * often results from the composition of several functions + * (derivationStrict, derivation, mkDerivation, mkPythonModule, etc.) + */ + e.addTrace(nullptr, hintfmt( + "while evaluating derivation '%s'\n" + " whose name attribute is located at %s", + drvName, pos), true); throw; } +} +static void derivationStrictInternal(EvalState & state, const std::string & +drvName, Bindings * attrs, Value & v) +{ /* Check whether attributes should be passed as a JSON file. */ + using nlohmann::json; std::optional<json> jsonObject; - attr = args[0]->attrs->find(state.sStructuredAttrs); - if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + auto attr = attrs->find(state.sStructuredAttrs); + if (attr != attrs->end() && + state.forceBool(*attr->value, noPos, + "while evaluating the `__structuredAttrs` " + "attribute passed to builtins.derivationStrict")) jsonObject = json::object(); /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; - attr = args[0]->attrs->find(state.sIgnoreNulls); - if (attr != args[0]->attrs->end()) - ignoreNulls = state.forceBool(*attr->value, pos); + attr = attrs->find(state.sIgnoreNulls); + if (attr != attrs->end()) + ignoreNulls = state.forceBool(*attr->value, noPos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ Derivation drv; @@ -1059,7 +1086,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * StringSet outputs; outputs.insert("out"); - for (auto & i : args[0]->attrs->lexicographicOrder(state.symbols)) { + for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; const std::string & key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); @@ -1070,7 +1097,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); }; @@ -1080,7 +1107,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (outputs.find(j) != outputs.end()) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("duplicate derivation output '%1%'", j), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); /* !!! Check whether j is a valid attribute name. */ @@ -1090,32 +1117,35 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (j == "drv") state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid derivation output name 'drv'" ), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); outputs.insert(j); } if (outputs.empty()) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("derivation cannot have an empty set of outputs"), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); }; try { + // This try-catch block adds context for most errors. + // Use this empty error context to signify that we defer to it. + const std::string_view context_below(""); if (ignoreNulls) { - state.forceValue(*i->value, pos); + state.forceValue(*i->value, noPos); if (i->value->type() == nNull) continue; } if (i->name == state.sContentAddressed) { - contentAddressed = state.forceBool(*i->value, pos); + contentAddressed = state.forceBool(*i->value, noPos, context_below); if (contentAddressed) settings.requireExperimentalFeature(Xp::CaDerivations); } else if (i->name == state.sImpure) { - isImpure = state.forceBool(*i->value, pos); + isImpure = state.forceBool(*i->value, noPos, context_below); if (isImpure) settings.requireExperimentalFeature(Xp::ImpureDerivations); } @@ -1123,9 +1153,11 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { - state.forceList(*i->value, pos); + state.forceList(*i->value, noPos, context_below); for (auto elem : i->value->listItems()) { - auto s = state.coerceToString(posDrvName, *elem, context, true).toOwned(); + auto s = state.coerceToString(noPos, *elem, context, + "while evaluating an element of the argument list", + true).toOwned(); drv.args.push_back(s); } } @@ -1138,29 +1170,29 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); + (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, noPos, context); if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, posDrvName); + drv.builder = state.forceString(*i->value, context, noPos, context_below); else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + drv.platform = state.forceStringNoCtx(*i->value, noPos, context_below); else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, posDrvName); + outputHash = state.forceStringNoCtx(*i->value, noPos, context_below); else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + outputHashAlgo = state.forceStringNoCtx(*i->value, noPos, context_below); else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + handleHashMode(state.forceStringNoCtx(*i->value, noPos, context_below)); else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, posDrvName); + state.forceList(*i->value, noPos, context_below); Strings ss; for (auto elem : i->value->listItems()) - ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); + ss.emplace_back(state.forceStringNoCtx(*elem, noPos, context_below)); handleOutputs(ss); } } else { - auto s = state.coerceToString(i->pos, *i->value, context, true).toOwned(); + auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = std::move(s); else if (i->name == state.sSystem) drv.platform = std::move(s); @@ -1174,9 +1206,9 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } catch (Error & e) { - e.addTrace(state.positions[posDrvName], - "while evaluating the attribute '%1%' of the derivation '%2%'", - key, drvName); + e.addTrace(state.positions[i->pos], + hintfmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName), + true); throw; } } @@ -1189,55 +1221,51 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & path : context) { - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - StorePathSet refs; - state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); - for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); - } - } - - /* Handle derivation outputs of the form ‘!<name>!<path>’. */ - else if (path.at(0) == '!') { - auto ctx = decodeContext(*state.store, path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(state.store->parseStorePath(path)); + for (auto & c_ : context) { + auto c = NixStringContextElem::parse(*state.store, c_); + std::visit(overloaded { + /* Since this allows the builder to gain access to every + path in the dependency graph of the derivation (including + all outputs), all paths in the graph must be added to + this derivation's list of inputs to ensure that they are + available when the builder runs. */ + [&](const NixStringContextElem::DrvDeep & d) { + /* !!! This doesn't work if readOnlyMode is set. */ + StorePathSet refs; + state.store->computeFSClosure(d.drvPath, refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (j.isDerivation()) + drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + } + }, + [&](const NixStringContextElem::Built & b) { + drv.inputDrvs[b.drvPath].insert(b.output); + }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(o.path); + }, + }, c.raw()); } /* Do we have all required attributes? */ if (drv.builder == "") state.debugThrowLastTrace(EvalError({ .msg = hintfmt("required attribute 'builder' missing"), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); if (drv.platform == "") state.debugThrowLastTrace(EvalError({ .msg = hintfmt("required attribute 'system' missing"), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); if (outputHash) { @@ -1248,7 +1276,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (outputs.size() != 1 || *(outputs.begin()) != "out") state.debugThrowLastTrace(Error({ .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] })); auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); @@ -1269,7 +1297,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (contentAddressed && isImpure) throw EvalError({ .msg = hintfmt("derivation cannot be both content-addressed and impure"), - .errPos = state.positions[posDrvName] + .errPos = state.positions[noPos] }); auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); @@ -1313,7 +1341,7 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * if (!h) throw AssertionError({ .msg = hintfmt("derivation produced no hash for output '%s'", i), - .errPos = state.positions[posDrvName], + .errPos = state.positions[noPos], }); auto outPath = state.store->makeOutputPath(i, *h, drvName); drv.env[i] = state.store->printStorePath(outPath); @@ -1346,11 +1374,12 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * drvHashes.lock()->insert_or_assign(drvPath, h); } - auto attrs = state.buildBindings(1 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + auto result = state.buildBindings(1 + drv.outputs.size()); + result.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) - mkOutputString(state, attrs, drvPath, drv, i); - v.mkAttrs(attrs); + mkOutputString(state, result, drvPath, drv, i); + + v.mkAttrs(result); } static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { @@ -1368,7 +1397,7 @@ static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { ‘out’. */ static void prim_placeholder(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); + v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.placeholder"))); } static RegisterPrimOp primop_placeholder({ @@ -1392,7 +1421,7 @@ static RegisterPrimOp primop_placeholder({ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); + Path path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); v.mkString(canonPath(path), context); } @@ -1423,7 +1452,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); PathSet context; - Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")); /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -1461,10 +1490,10 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to - access an unauthorized path). */ + throw if the path isn’t allowed, but just return false (and we + can’t just catch the exception here because we still want to + throw if something in the evaluation of `*args[0]` tries to + access an unauthorized path). */ auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); try { @@ -1493,7 +1522,9 @@ static RegisterPrimOp primop_pathExists({ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, false, false)), context); + v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.baseNameOf", + false, false)), context); } static RegisterPrimOp primop_baseNameOf({ @@ -1513,7 +1544,9 @@ static RegisterPrimOp primop_baseNameOf({ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto path = state.coerceToString(pos, *args[0], context, false, false); + auto path = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.dirOf", + false, false); auto dir = dirOf(*path); if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); } @@ -1564,28 +1597,24 @@ static RegisterPrimOp primop_readFile({ which are desugared to 'findFile __nixPath "x"'. */ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.findFile"); SearchPath searchPath; for (auto v2 : args[0]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile"); std::string prefix; Bindings::iterator i = v2->attrs->find(state.sPrefix); if (i != v2->attrs->end()) - prefix = state.forceStringNoCtx(*i->value, pos); + prefix = state.forceStringNoCtx(*i->value, pos, "while evaluating the `prefix` attribute of an element of the list passed to builtins.findFile"); - i = getAttr( - state, - "findFile", - state.sPath, - v2->attrs, - pos - ); + i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); PathSet context; - auto path = state.coerceToString(pos, *i->value, context, false, false).toOwned(); + auto path = state.coerceToString(pos, *i->value, context, + "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", + false, false).toOwned(); try { auto rewrites = state.realiseContext(context); @@ -1600,7 +1629,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V searchPath.emplace_back(prefix, path); } - auto path = state.forceStringNoCtx(*args[1], pos); + auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } @@ -1614,7 +1643,7 @@ static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos); + auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); std::optional<HashType> ht = parseHashType(type); if (!ht) state.debugThrowLastTrace(Error({ @@ -1638,23 +1667,73 @@ static RegisterPrimOp primop_hashFile({ .fun = prim_hashFile, }); + +/* Stringize a directory entry enum. Used by `readFileType' and `readDir'. */ +static const char * dirEntTypeToString(unsigned char dtType) +{ + /* Enum DT_(DIR|LNK|REG|UNKNOWN) */ + switch(dtType) { + case DT_REG: return "regular"; break; + case DT_DIR: return "directory"; break; + case DT_LNK: return "symlink"; break; + default: return "unknown"; break; + } + return "unknown"; /* Unreachable */ +} + + +static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + auto path = realisePath(state, pos, *args[0]); + /* Retrieve the directory entry type and stringize it. */ + v.mkString(dirEntTypeToString(getFileType(path))); +} + +static RegisterPrimOp primop_readFileType({ + .name = "__readFileType", + .args = {"p"}, + .doc = R"( + Determine the directory entry type of a filesystem node, being + one of "directory", "regular", "symlink", or "unknown". + )", + .fun = prim_readFileType, +}); + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); + // Retrieve directory entries for all nodes in a directory. + // This is similar to `getFileType` but is optimized to reduce system calls + // on many systems. DirEntries entries = readDirectory(path); auto attrs = state.buildBindings(entries.size()); + // If we hit unknown directory entry types we may need to fallback to + // using `getFileType` on some systems. + // In order to reduce system calls we make each lookup lazy by using + // `builtins.readFileType` application. + Value * readFileType = nullptr; + for (auto & ent : entries) { - if (ent.type == DT_UNKNOWN) - ent.type = getFileType(path + "/" + ent.name); - attrs.alloc(ent.name).mkString( - ent.type == DT_REG ? "regular" : - ent.type == DT_DIR ? "directory" : - ent.type == DT_LNK ? "symlink" : - "unknown"); + auto & attr = attrs.alloc(ent.name); + if (ent.type == DT_UNKNOWN) { + // Some filesystems or operating systems may not be able to return + // detailed node info quickly in this case we produce a thunk to + // query the file type lazily. + auto epath = state.allocValue(); + Path path2 = path + "/" + ent.name; + epath->mkString(path2); + if (!readFileType) + readFileType = &state.getBuiltin("readFileType"); + attr.mkApp(readFileType, epath); + } else { + // This branch of the conditional is much more likely. + // Here we just stringize the directory entry type. + attr.mkString(dirEntTypeToString(ent.type)); + } } v.mkAttrs(attrs); @@ -1821,7 +1900,7 @@ static RegisterPrimOp primop_toJSON({ /* Parse a JSON string to a value. */ static void prim_fromJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto s = state.forceStringNoCtx(*args[0], pos); + auto s = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.fromJSON"); try { parseJSON(state, s, v); } catch (JSONParseError &e) { @@ -1850,8 +1929,8 @@ static RegisterPrimOp primop_fromJSON({ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - std::string name(state.forceStringNoCtx(*args[0], pos)); - std::string contents(state.forceString(*args[1], context, pos)); + std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); + std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; @@ -1887,8 +1966,7 @@ static RegisterPrimOp primop_toFile({ path. The file has suffix *name*. This file can be used as an input to derivations. One application is to write builders “inline”. For instance, the following Nix expression combines the - [Nix expression for GNU Hello](expression-syntax.md) and its - [build script](build-script.md) into one file: + Nix expression for GNU Hello and its build script into one file: ```nix { stdenv, fetchurl, perl }: @@ -1931,8 +2009,8 @@ static RegisterPrimOp primop_toFile({ "; ``` - Note that `${configFile}` is an - [antiquotation](language-values.md), so the result of the + Note that `${configFile}` is a + [string interpolation](@docroot@/language/values.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. @@ -2009,7 +2087,7 @@ static void addPath( Value res; state.callFunction(*filterFun, 2, args, res, pos); - return state.forceBool(res, pos); + return state.forceBool(res, pos, "while evaluating the return value of the path filter function"); }) : defaultPathFilter; std::optional<StorePath> expectedStorePath; @@ -2035,17 +2113,8 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - - state.forceValue(*args[0], pos); - if (args[0]->type() != nFunction) - state.debugThrowLastTrace(TypeError({ - .msg = hintfmt( - "first argument in call to 'filterSource' is not a function but %1%", - showType(*args[0])), - .errPos = state.positions[pos] - })); - + Path path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } @@ -2106,7 +2175,7 @@ static RegisterPrimOp primop_filterSource({ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.path"); Path path; std::string name; Value * filterFun = nullptr; @@ -2117,16 +2186,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value for (auto & attr : *args[0]->attrs) { auto n = state.symbols[attr.name]; if (n == "path") - path = state.coerceToPath(attr.pos, *attr.value, context); + path = state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the `path` attribute passed to builtins.path"); else if (attr.name == state.sName) - name = state.forceStringNoCtx(*attr.value, attr.pos); - else if (n == "filter") { - state.forceValue(*attr.value, pos); - filterFun = attr.value; - } else if (n == "recursive") - method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos) }; + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path"); + else if (n == "filter") + state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); + else if (n == "recursive") + method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256); else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), @@ -2135,7 +2203,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value } if (path.empty()) state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'path' required"), + .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"), .errPos = state.positions[pos] })); if (name.empty()) @@ -2189,7 +2257,7 @@ static RegisterPrimOp primop_path({ strings. */ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrNames"); state.mkList(v, args[0]->attrs->size()); @@ -2216,7 +2284,7 @@ static RegisterPrimOp primop_attrNames({ order as attrNames. */ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.attrValues"); state.mkList(v, args[0]->attrs->size()); @@ -2248,14 +2316,13 @@ static RegisterPrimOp primop_attrValues({ /* Dynamic version of the `.' operator. */ void prim_getAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getAttr"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.getAttr"); Bindings::iterator i = getAttr( state, - "getAttr", state.symbols.create(attr), args[1]->attrs, - pos + "in the attribute set under consideration" ); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[i->pos]++; @@ -2278,8 +2345,8 @@ static RegisterPrimOp primop_getAttr({ /* Return position information of the specified attribute. */ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.unsafeGetAttrPos"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.unsafeGetAttrPos"); Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) v.mkNull(); @@ -2296,8 +2363,8 @@ static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { /* Dynamic version of the `?' operator. */ static void prim_hasAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attr = state.forceStringNoCtx(*args[0], pos); - state.forceAttrs(*args[1], pos); + auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hasAttr"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.hasAttr"); v.mkBool(args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); } @@ -2330,8 +2397,8 @@ static RegisterPrimOp primop_isAttrs({ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - state.forceList(*args[1], pos); + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.removeAttrs"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.removeAttrs"); /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference @@ -2339,7 +2406,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args boost::container::small_vector<Attr, 64> names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { - state.forceStringNoCtx(*elem, pos); + state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); names.emplace_back(state.symbols.create(elem->string.s), nullptr); } std::sort(names.begin(), names.end()); @@ -2378,34 +2445,22 @@ static RegisterPrimOp primop_removeAttrs({ name, the first takes precedence. */ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the argument passed to builtins.listToAttrs"); auto attrs = state.buildBindings(args[0]->listSize()); std::set<Symbol> seen; for (auto v2 : args[0]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.listToAttrs"); - Bindings::iterator j = getAttr( - state, - "listToAttrs", - state.sName, - v2->attrs, - pos - ); + Bindings::iterator j = getAttr(state, state.sName, v2->attrs, "in a {name=...; value=...;} pair"); - auto name = state.forceStringNoCtx(*j->value, j->pos); + auto name = state.forceStringNoCtx(*j->value, j->pos, "while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs"); auto sym = state.symbols.create(name); if (seen.insert(sym).second) { - Bindings::iterator j2 = getAttr( - state, - "listToAttrs", - state.sValue, - v2->attrs, - pos - ); + Bindings::iterator j2 = getAttr(state, state.sValue, v2->attrs, "in a {name=...; value=...;} pair"); attrs.insert(sym, j2->value, j2->pos); } } @@ -2446,15 +2501,65 @@ static RegisterPrimOp primop_listToAttrs({ static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); - state.forceAttrs(*args[1], pos); - - auto attrs = state.buildBindings(std::min(args[0]->attrs->size(), args[1]->attrs->size())); - - for (auto & i : *args[0]->attrs) { - Bindings::iterator j = args[1]->attrs->find(i.name); - if (j != args[1]->attrs->end()) - attrs.insert(*j); + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.intersectAttrs"); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.intersectAttrs"); + + Bindings &left = *args[0]->attrs; + Bindings &right = *args[1]->attrs; + + auto attrs = state.buildBindings(std::min(left.size(), right.size())); + + // The current implementation has good asymptotic complexity and is reasonably + // simple. Further optimization may be possible, but does not seem productive, + // considering the state of eval performance in 2022. + // + // I have looked for reusable and/or standard solutions and these are my + // findings: + // + // STL + // === + // std::set_intersection is not suitable, as it only performs a simultaneous + // linear scan; not taking advantage of random access. This is O(n + m), so + // linear in the largest set, which is not acceptable for callPackage in Nixpkgs. + // + // Simultaneous scan, with alternating simple binary search + // === + // One alternative algorithm scans the attrsets simultaneously, jumping + // forward using `lower_bound` in case of inequality. This should perform + // well on very similar sets, having a local and predictable access pattern. + // On dissimilar sets, it seems to need more comparisons than the current + // algorithm, as few consecutive attrs match. `lower_bound` could take + // advantage of the decreasing remaining search space, but this causes + // the medians to move, which can mean that they don't stay in the cache + // like they would with the current naive `find`. + // + // Double binary search + // === + // The optimal algorithm may be "Double binary search", which doesn't + // scan at all, but rather divides both sets simultaneously. + // See "Fast Intersection Algorithms for Sorted Sequences" by Baeza-Yates et al. + // https://cs.uwaterloo.ca/~ajsaling/papers/intersection_alg_app10.pdf + // The only downsides I can think of are not having a linear access pattern + // for similar sets, and having to maintain a more intricate algorithm. + // + // Adaptive + // === + // Finally one could run try a simultaneous scan, count misses and fall back + // to double binary search when the counter hit some threshold and/or ratio. + + if (left.size() < right.size()) { + for (auto & l : left) { + Bindings::iterator r = right.find(l.name); + if (r != right.end()) + attrs.insert(*r); + } + } + else { + for (auto & r : right) { + Bindings::iterator l = left.find(r.name); + if (l != left.end()) + attrs.insert(r); + } } v.mkAttrs(attrs.alreadySorted()); @@ -2466,20 +2571,22 @@ static RegisterPrimOp primop_intersectAttrs({ .doc = R"( Return a set consisting of the attributes in the set *e2* which have the same name as some attribute in *e1*. + + Performs in O(*n* log *m*) where *n* is the size of the smaller set and *m* the larger set's size. )", .fun = prim_intersectAttrs, }); static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); - state.forceList(*args[1], pos); + auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); Value * res[args[1]->listSize()]; unsigned int found = 0; for (auto v2 : args[1]->listItems()) { - state.forceAttrs(*v2, pos); + state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); Bindings::iterator i = v2->attrs->find(attrName); if (i != v2->attrs->end()) res[found++] = i->value; @@ -2552,7 +2659,7 @@ static RegisterPrimOp primop_functionArgs({ /* */ static void prim_mapAttrs(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.mapAttrs"); auto attrs = state.buildBindings(args[1]->attrs->size()); @@ -2593,21 +2700,16 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg std::map<Symbol, std::pair<size_t, Value * *>> attrsSeen; - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); const auto listSize = args[1]->listSize(); const auto listElems = args[1]->listElems(); for (unsigned int n = 0; n < listSize; ++n) { Value * vElem = listElems[n]; - try { - state.forceAttrs(*vElem, noPos); - for (auto & attr : *vElem->attrs) - attrsSeen[attr.name].first++; - } catch (TypeError & e) { - e.addTrace(state.positions[pos], hintfmt("while invoking '%s'", "zipAttrsWith")); - state.debugThrowLastTrace(e); - } + state.forceAttrs(*vElem, noPos, "while evaluating a value of the list passed as second argument to builtins.zipAttrsWith"); + for (auto & attr : *vElem->attrs) + attrsSeen[attr.name].first++; } auto attrs = state.buildBindings(attrsSeen.size()); @@ -2691,7 +2793,7 @@ static RegisterPrimOp primop_isList({ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) { - state.forceList(list, pos); + state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); if (n < 0 || (unsigned int) n >= list.listSize()) state.debugThrowLastTrace(Error({ .msg = hintfmt("list index %1% is out of bounds", n), @@ -2704,7 +2806,7 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); + elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v); } static RegisterPrimOp primop_elemAt({ @@ -2739,7 +2841,7 @@ static RegisterPrimOp primop_head({ don't want to use it! */ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); if (args[0]->listSize() == 0) state.debugThrowLastTrace(Error({ .msg = hintfmt("'tail' called on an empty list"), @@ -2770,10 +2872,16 @@ static RegisterPrimOp primop_tail({ /* Apply a function to every element of a list. */ static void prim_map(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.map"); - state.mkList(v, args[1]->listSize()); + if (args[1]->listSize() == 0) { + v = *args[1]; + return; + } + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.map"); + + state.mkList(v, args[1]->listSize()); for (unsigned int n = 0; n < v.listSize(); ++n) (v.listElems()[n] = state.allocValue())->mkApp( args[0], args[1]->listElems()[n]); @@ -2800,8 +2908,14 @@ static RegisterPrimOp primop_map({ returns true. */ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.filter"); + + if (args[1]->listSize() == 0) { + v = *args[1]; + return; + } + + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); // FIXME: putting this on the stack is risky. Value * vs[args[1]->listSize()]; @@ -2811,7 +2925,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val for (unsigned int n = 0; n < args[1]->listSize(); ++n) { Value res; state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); - if (state.forceBool(res, pos)) + if (state.forceBool(res, pos, "while evaluating the return value of the filtering function passed to builtins.filter")) vs[k++] = args[1]->listElems()[n]; else same = false; @@ -2839,9 +2953,9 @@ static RegisterPrimOp primop_filter({ static void prim_elem(EvalState & state, const PosIdx pos, Value * * args, Value & v) { bool res = false; - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.elem"); for (auto elem : args[1]->listItems()) - if (state.eqValues(*args[0], *elem)) { + if (state.eqValues(*args[0], *elem, pos, "while searching for the presence of the given element in the list")) { res = true; break; } @@ -2861,8 +2975,8 @@ static RegisterPrimOp primop_elem({ /* Concatenate a list of lists. */ static void prim_concatLists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); - state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.concatLists"); + state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos, "while evaluating a value of the list passed to builtins.concatLists"); } static RegisterPrimOp primop_concatLists({ @@ -2877,7 +2991,7 @@ static RegisterPrimOp primop_concatLists({ /* Return the length of a list. This is an O(1) time operation. */ static void prim_length(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.length"); v.mkInt(args[0]->listSize()); } @@ -2894,8 +3008,8 @@ static RegisterPrimOp primop_length({ right. The operator is applied strictly. */ static void prim_foldlStrict(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[2], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.foldlStrict"); + state.forceList(*args[2], pos, "while evaluating the third argument passed to builtins.foldlStrict"); if (args[2]->listSize()) { Value * vCur = args[1]; @@ -2918,22 +3032,22 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. + The return value of each application of `op` is evaluated immediately, + even for intermediate values. )", .fun = prim_foldlStrict, }); static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); + state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos); + bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); if (res == any) { v.mkBool(any); return; @@ -2976,16 +3090,16 @@ static RegisterPrimOp primop_all({ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos); + auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); if (len < 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot create list of size %1%", len), - .errPos = state.positions[pos] - })); + state.error("cannot create list of size %1%", len).debugThrow<EvalError>(); - state.mkList(v, len); + // More strict than striclty (!) necessary, but acceptable + // as evaluating map without accessing any values makes little sense. + state.forceFunction(*args[0], noPos, "while evaluating the first argument passed to builtins.genList"); + state.mkList(v, len); for (unsigned int n = 0; n < (unsigned int) len; ++n) { auto arg = state.allocValue(); arg->mkInt(n); @@ -3014,10 +3128,16 @@ static void prim_lessThan(EvalState & state, const PosIdx pos, Value * * args, V static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.sort"); auto len = args[1]->listSize(); + if (len == 0) { + v = *args[1]; + return; + } + + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.sort"); + state.mkList(v, len); for (unsigned int n = 0; n < len; ++n) { state.forceValue(*args[1]->listElems()[n], pos); @@ -3027,13 +3147,15 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ + /* TODO: (layus) this is absurd. An optimisation like this + should be outside the lambda creation */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues(state)(a, b); + return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b); Value * vs[] = {a, b}; Value vBool; - state.callFunction(*args[0], 2, vs, vBool, pos); - return state.forceBool(vBool, pos); + state.callFunction(*args[0], 2, vs, vBool, noPos); + return state.forceBool(vBool, pos, "while evaluating the return value of the sorting function passed to builtins.sort"); }; /* FIXME: std::sort can segfault if the comparator is not a strict @@ -3065,8 +3187,8 @@ static RegisterPrimOp primop_sort({ static void prim_partition(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.partition"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.partition"); auto len = args[1]->listSize(); @@ -3077,7 +3199,7 @@ static void prim_partition(EvalState & state, const PosIdx pos, Value * * args, state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); - if (state.forceBool(res, pos)) + if (state.forceBool(res, pos, "while evaluating the return value of the partition function passed to builtins.partition")) right.push_back(vElem); else wrong.push_back(vElem); @@ -3125,15 +3247,15 @@ static RegisterPrimOp primop_partition({ static void prim_groupBy(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.groupBy"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.groupBy"); ValueVectorMap attrs; for (auto vElem : args[1]->listItems()) { Value res; state.callFunction(*args[0], *vElem, res, pos); - auto name = state.forceStringNoCtx(res, pos); + auto name = state.forceStringNoCtx(res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy"); auto sym = state.symbols.create(name); auto vector = attrs.try_emplace(sym, ValueVector()).first; vector->second.push_back(vElem); @@ -3177,8 +3299,8 @@ static RegisterPrimOp primop_groupBy({ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceFunction(*args[0], pos); - state.forceList(*args[1], pos); + state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.concatMap"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); Value lists[nrLists]; @@ -3187,12 +3309,7 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - try { - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos))); - } catch (TypeError &e) { - e.addTrace(state.positions[pos], hintfmt("while invoking '%s'", "concatMap")); - state.debugThrowLastTrace(e); - } + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); len += lists[n].listSize(); } @@ -3227,9 +3344,11 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); else - v.mkInt(state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition") + + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition")); } static RegisterPrimOp primop_add({ @@ -3246,9 +3365,11 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") + - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); else - v.mkInt(state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction") + - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction")); } static RegisterPrimOp primop_sub({ @@ -3265,9 +3386,11 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - v.mkFloat(state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") + * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); else - v.mkInt(state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication") + * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication")); } static RegisterPrimOp primop_mul({ @@ -3284,7 +3407,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - NixFloat f2 = state.forceFloat(*args[1], pos); + NixFloat f2 = state.forceFloat(*args[1], pos, "while evaluating the second operand of the division"); if (f2 == 0) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("division by zero"), @@ -3292,10 +3415,10 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value })); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { - v.mkFloat(state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first operand of the division") / f2); } else { - NixInt i1 = state.forceInt(*args[0], pos); - NixInt i2 = state.forceInt(*args[1], pos); + NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); + NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) state.debugThrowLastTrace(EvalError({ @@ -3318,7 +3441,8 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd") + & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd")); } static RegisterPrimOp primop_bitAnd({ @@ -3332,7 +3456,8 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr") + | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr")); } static RegisterPrimOp primop_bitOr({ @@ -3346,7 +3471,8 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor") + ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor")); } static RegisterPrimOp primop_bitXor({ @@ -3362,7 +3488,8 @@ static void prim_lessThan(EvalState & state, const PosIdx pos, Value * * args, V { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp{state}; + // pos is exact here, no need for a message. + CompareValues comp(state, noPos, ""); v.mkBool(comp(args[0], args[1])); } @@ -3389,7 +3516,9 @@ static RegisterPrimOp primop_lessThan({ static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto s = state.coerceToString(pos, *args[0], context, true, false); + auto s = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.toString", + true, false); v.mkString(*s, context); } @@ -3423,10 +3552,10 @@ static RegisterPrimOp primop_toString({ non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos); - int len = state.forceInt(*args[1], pos); + int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); PathSet context; - auto s = state.coerceToString(pos, *args[2], context); + auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); if (start < 0) state.debugThrowLastTrace(EvalError({ @@ -3460,7 +3589,7 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); v.mkInt(s->size()); } @@ -3477,7 +3606,7 @@ static RegisterPrimOp primop_stringLength({ /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos); + auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); std::optional<HashType> ht = parseHashType(type); if (!ht) state.debugThrowLastTrace(Error({ @@ -3486,7 +3615,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, })); PathSet context; // discarded - auto s = state.forceString(*args[1], context, pos); + auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); v.mkString(hashString(*ht, s).to_string(Base16, false)); } @@ -3525,14 +3654,14 @@ std::shared_ptr<RegexCache> makeRegexCache() void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto re = state.forceStringNoCtx(*args[0], pos); + auto re = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.match"); try { auto regex = state.regexCache->get(re); PathSet context; - const auto str = state.forceString(*args[1], context, pos); + const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; if (!std::regex_match(str.begin(), str.end(), match, regex)) { @@ -3605,14 +3734,14 @@ static RegisterPrimOp primop_match({ non-matching parts interleaved by the lists of the matching groups. */ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto re = state.forceStringNoCtx(*args[0], pos); + auto re = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.split"); try { auto regex = state.regexCache->get(re); PathSet context; - const auto str = state.forceString(*args[1], context, pos); + const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); auto end = std::cregex_iterator(); @@ -3710,8 +3839,8 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * { PathSet context; - auto sep = state.forceString(*args[0], context, pos); - state.forceList(*args[1], pos); + auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); + state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); std::string res; res.reserve((args[1]->listSize() + 32) * sep.size()); @@ -3719,7 +3848,7 @@ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += *state.coerceToString(pos, *elem, context); + res += *state.coerceToString(pos, *elem, context, "while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep"); } v.mkString(res, context); @@ -3738,29 +3867,26 @@ static RegisterPrimOp primop_concatStringsSep({ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos); - state.forceList(*args[1], pos); + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.replaceStrings"); + state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.replaceStrings"); if (args[0]->listSize() != args[1]->listSize()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), - .errPos = state.positions[pos] - })); + state.error("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths").atPos(pos).debugThrow<EvalError>(); std::vector<std::string> from; from.reserve(args[0]->listSize()); for (auto elem : args[0]->listItems()) - from.emplace_back(state.forceString(*elem, pos)); + from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); std::vector<std::pair<std::string, PathSet>> to; to.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { PathSet ctx; - auto s = state.forceString(*elem, ctx, pos); + auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); to.emplace_back(s, std::move(ctx)); } PathSet context; - auto s = state.forceString(*args[2], context, pos); + auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; // Loops one past last character to handle the case where 'from' contains an empty string. @@ -3818,7 +3944,7 @@ static RegisterPrimOp primop_replaceStrings({ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto name = state.forceStringNoCtx(*args[0], pos); + auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName"); DrvName parsed(name); auto attrs = state.buildBindings(2); attrs.alloc(state.sName).mkString(parsed.name); @@ -3842,8 +3968,8 @@ static RegisterPrimOp primop_parseDrvName({ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto version1 = state.forceStringNoCtx(*args[0], pos); - auto version2 = state.forceStringNoCtx(*args[1], pos); + auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); + auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); v.mkInt(compareVersions(version1, version2)); } @@ -3862,7 +3988,7 @@ static RegisterPrimOp primop_compareVersions({ static void prim_splitVersion(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto version = state.forceStringNoCtx(*args[0], pos); + auto version = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.splitVersion"); auto iter = version.cbegin(); Strings components; while (iter != version.cend()) { @@ -4018,7 +4144,7 @@ void EvalState::createBaseEnv() // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), foFile, derivationNixPath, "/", staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationNixPath, "/", staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 979136984..db43e5771 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -8,7 +8,7 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } @@ -18,7 +18,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringCo static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - state.forceString(*args[0], context, pos); + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } @@ -34,11 +34,18 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto s = state.coerceToString(pos, *args[0], context); + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + for (auto && p : context) { + auto c = NixStringContextElem::parse(*state.store, p); + if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) { + context2.emplace(state.store->printStorePath(ptr->drvPath)); + } else { + /* Can reuse original item */ + context2.emplace(std::move(p)); + } + } v.mkString(*s, context2); } @@ -73,35 +80,21 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, Strings outputs; }; PathSet context; - state.forceString(*args[0], context, pos); - auto contextInfos = std::map<Path, ContextInfo>(); + state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); + auto contextInfos = std::map<StorePath, ContextInfo>(); for (const auto & p : context) { - Path drv; - std::string output; - const Path * path = &p; - if (p.at(0) == '=') { - drv = std::string(p, 1); - path = &drv; - } else if (p.at(0) == '!') { - NixStringContextElem ctx = decodeContext(*state.store, p); - drv = state.store->printStorePath(ctx.first); - output = ctx.second; - path = &drv; - } - auto isPath = drv.empty(); - auto isAllOutputs = (!drv.empty()) && output.empty(); - - auto iter = contextInfos.find(*path); - if (iter == contextInfos.end()) { - contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); - } else { - if (isPath) - iter->second.path = true; - else if (isAllOutputs) - iter->second.allOutputs = true; - else - iter->second.outputs.emplace_back(std::move(output)); - } + NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + std::visit(overloaded { + [&](NixStringContextElem::DrvDeep & d) { + contextInfos[d.drvPath].allOutputs = true; + }, + [&](NixStringContextElem::Built & b) { + contextInfos[b.drvPath].outputs.emplace_back(std::move(b.output)); + }, + [&](NixStringContextElem::Opaque & o) { + contextInfos[o.path].path = true; + }, + }, ctx.raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -120,7 +113,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, for (const auto & [i, output] : enumerate(info.second.outputs)) (outputsVal.listElems()[i] = state.allocValue())->mkString(output); } - attrs.alloc(info.first).mkAttrs(infoAttrs); + attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); } v.mkAttrs(attrs); @@ -137,9 +130,9 @@ static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { PathSet context; - auto orig = state.forceString(*args[0], context, pos); + auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); - state.forceAttrs(*args[1], pos); + state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); auto sPath = state.symbols.create("path"); auto sAllOutputs = state.symbols.create("allOutputs"); @@ -147,24 +140,24 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar const auto & name = state.symbols[i.name]; if (!state.store->isStorePath(name)) throw EvalError({ - .msg = hintfmt("Context key '%s' is not a store path", name), + .msg = hintfmt("context key '%s' is not a store path", name), .errPos = state.positions[i.pos] }); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(name)); - state.forceAttrs(*i.value, i.pos); + state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, iter->pos)) + if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) context.emplace(name); } iter = i.value->attrs->find(sAllOutputs); if (iter != i.value->attrs->end()) { - if (state.forceBool(*iter->value, iter->pos)) { + if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) { if (!isDerivation(name)) { throw EvalError({ - .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", name), + .msg = hintfmt("tried to add all-outputs context of %s, which is not a derivation, to a string", name), .errPos = state.positions[i.pos] }); } @@ -174,15 +167,15 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar iter = i.value->attrs->find(state.sOutputs); if (iter != i.value->attrs->end()) { - state.forceList(*iter->value, iter->pos); + state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context"); if (iter->value->listSize() && !isDerivation(name)) { throw EvalError({ - .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", name), + .msg = hintfmt("tried to add derivation output context of %s, which is not a derivation, to a string", name), .errPos = state.positions[i.pos] }); } for (auto elem : iter->value->listItems()) { - auto outputName = state.forceStringNoCtx(*elem, iter->pos); + auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.insert(concatStrings("!", outputName, "!", name)); } } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 662c9652e..0dfa97fa3 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -7,7 +7,7 @@ namespace nix { static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional<std::string> fromStoreUrl; std::optional<StorePath> fromPath; @@ -19,7 +19,8 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg if (attrName == "fromPath") { PathSet context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context); + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, + "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); } else if (attrName == "toPath") { @@ -27,12 +28,14 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { PathSet context; - toPath = state.coerceToStorePath(attr.pos, *attr.value, context); + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, + "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); } } else if (attrName == "fromStore") - fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos); + fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, + "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure"); else throw Error({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 249c0934e..c41bd60b6 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -19,23 +19,23 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); - for (auto & attr : *args[0]->attrs) { std::string_view n(state.symbols[attr.name]); if (n == "url") - url = state.coerceToString(attr.pos, *attr.value, context, false, false).toOwned(); + url = state.coerceToString(attr.pos, *attr.value, context, + "while evaluating the `url` attribute passed to builtins.fetchMercurial", + false, false).toOwned(); else if (n == "rev") { // Ugly: unlike fetchGit, here the "rev" attribute can // be both a revision or a branch/tag name. - auto value = state.forceStringNoCtx(*attr.value, attr.pos); + auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) rev = Hash::parseAny(value, htSHA1); else ref = value; } else if (n == "name") - name = state.forceStringNoCtx(*attr.value, attr.pos); + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.fetchMercurial"); else throw EvalError({ .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]), @@ -50,7 +50,9 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a }); } else - url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); + url = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to builtins.fetchMercurial", + false, false).toOwned(); // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 84e7f5c02..2e924c302 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -102,7 +102,7 @@ static void fetchTree( state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchTree"); fetchers::Attrs attrs; @@ -112,7 +112,7 @@ static void fetchTree( .msg = hintfmt("unexpected attribute 'type'"), .errPos = state.positions[pos] })); - type = state.forceStringNoCtx(*aType->value, aType->pos); + type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree"); } else if (!type) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), @@ -125,7 +125,7 @@ static void fetchTree( if (attr.name == state.sType) continue; state.forceValue(*attr.value, attr.pos); if (attr.value->type() == nPath || attr.value->type() == nString) { - auto s = state.coerceToString(attr.pos, *attr.value, context, false, false).toOwned(); + auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], state.symbols[attr.name] == "url" ? type == "git" @@ -151,7 +151,9 @@ static void fetchTree( input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - auto url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); + auto url = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to the fetcher", + false, false).toOwned(); if (type == "git") { fetchers::Attrs attrs; @@ -195,16 +197,14 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (args[0]->type() == nAttrs) { - state.forceAttrs(*args[0], pos); - for (auto & attr : *args[0]->attrs) { std::string_view n(state.symbols[attr.name]); if (n == "url") - url = state.forceStringNoCtx(*attr.value, attr.pos); + url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, attr.pos); + name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("unsupported argument '%s' to '%s'", n, who), @@ -218,9 +218,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v .errPos = state.positions[pos] })); } else - url = state.forceStringNoCtx(*args[0], pos); + url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); - url = resolveUri(*url); + if (who == "fetchTarball") + url = evalSettings.resolvePseudoUrl(*url); state.checkURI(*url); @@ -455,6 +456,17 @@ static RegisterPrimOp primop_fetchGit({ > **Note** > > This behavior is disabled in *Pure evaluation mode*. + + - To fetch the content of a checked-out work directory: + + ```nix + builtins.fetchGit ./work-dir + ``` + + If the URL points to a local directory, and no `ref` or `rev` is + given, `fetchGit` will use the current content of the checked-out + files, even if they are not committed or added to Git's index. It will + only consider files added to the Git repository, as listed by `git ls-files`. )", .fun = prim_fetchGit, }); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 9753e2ac9..8a5231781 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -7,7 +7,7 @@ namespace nix { static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) { - auto toml = state.forceStringNoCtx(*args[0], pos); + auto toml = state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.fromTOML"); std::istringstream tomlStream(std::string{toml}); diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc new file mode 100644 index 000000000..24e95ac39 --- /dev/null +++ b/src/libexpr/tests/error_traces.cc @@ -0,0 +1,1298 @@ +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +#include "tests/libexpr.hh" + +namespace nix { + + using namespace testing; + + // Testing eval of PrimOp's + class ErrorTraceTest : public LibExprTest { }; + + TEST_F(ErrorTraceTest, TraceBuilder) { + ASSERT_THROW( + state.error("Not much").debugThrow<EvalError>(), + EvalError + ); + + ASSERT_THROW( + state.error("Not much").withTrace(noPos, "No more").debugThrow<EvalError>(), + EvalError + ); + + ASSERT_THROW( + try { + try { + state.error("Not much").withTrace(noPos, "No more").debugThrow<EvalError>(); + } catch (Error & e) { + e.addTrace(state.positions[noPos], "Something", ""); + throw; + } + } catch (BaseError & e) { + ASSERT_EQ(PrintToString(e.info().msg), + PrintToString(hintfmt("Not much"))); + auto trace = e.info().traces.rbegin(); + ASSERT_EQ(e.info().traces.size(), 2); + ASSERT_EQ(PrintToString(trace->hint), + PrintToString(hintfmt("No more"))); + trace++; + ASSERT_EQ(PrintToString(trace->hint), + PrintToString(hintfmt("Something"))); + throw; + } + , EvalError + ); + } + + TEST_F(ErrorTraceTest, NestedThrows) { + try { + state.error("Not much").withTrace(noPos, "No more").debugThrow<EvalError>(); + } catch (BaseError & e) { + try { + state.error("Not much more").debugThrow<EvalError>(); + } catch (Error & e2) { + e.addTrace(state.positions[noPos], "Something", ""); + //e2.addTrace(state.positions[noPos], "Something", ""); + ASSERT_TRUE(e.info().traces.size() == 2); + ASSERT_TRUE(e2.info().traces.size() == 0); + ASSERT_FALSE(&e.info() == &e2.info()); + } + } + } + +#define ASSERT_TRACE1(args, type, message) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + +#define ASSERT_TRACE2(args, type, message, context) \ + ASSERT_THROW( \ + std::string expr(args); \ + std::string name = expr.substr(0, expr.find(" ")); \ + try { \ + Value v = eval("builtins." args); \ + state.forceValueDeep(v); \ + } catch (BaseError & e) { \ + ASSERT_EQ(PrintToString(e.info().msg), \ + PrintToString(message)); \ + ASSERT_EQ(e.info().traces.size(), 2) << "while testing " args << std::endl << e.what(); \ + auto trace = e.info().traces.rbegin(); \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(context)); \ + ++trace; \ + ASSERT_EQ(PrintToString(trace->hint), \ + PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + throw; \ + } \ + , type \ + ) + + TEST_F(ErrorTraceTest, genericClosure) { + ASSERT_TRACE2("genericClosure 1", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure {}", + TypeError, + hintfmt("attribute '%s' missing", "startSet"), + hintfmt("in the attrset passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = 1; }", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = true; }", + TypeError, + hintfmt("value is %s while a function was expected", "a Boolean"), + hintfmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: true; }", + TypeError, + hintfmt("value is %s while a list was expected", "a Boolean"), + hintfmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ true ]; }", + TypeError, + hintfmt("value is %s while a set was expected", "a Boolean"), + hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ {} ]; }", + TypeError, + hintfmt("attribute '%s' missing", "key"), + hintfmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); + + ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [{ key = ''a''; }]; }", + EvalError, + hintfmt("cannot compare %s with %s", "a string", "an integer"), + hintfmt("while comparing the `key` attributes of two genericClosure elements")); + + ASSERT_TRACE2("genericClosure { startSet = [ true ]; operator = item: [{ key = ''a''; }]; }", + TypeError, + hintfmt("value is %s while a set was expected", "a Boolean"), + hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + + } + + + TEST_F(ErrorTraceTest, replaceStrings) { + ASSERT_TRACE2("replaceStrings 0 0 {}", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [] 0 {}", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the second argument passed to builtins.replaceStrings")); + + ASSERT_TRACE1("replaceStrings [ 0 ] [] {}", + EvalError, + hintfmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); + + ASSERT_TRACE2("replaceStrings [ 1 ] [ \"new\" ] {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}", + TypeError, + hintfmt("value is %s while a string was expected", "a Boolean"), + hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); + + ASSERT_TRACE2("replaceStrings [ \"old\" ] [ \"new\" ] {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the third argument passed to builtins.replaceStrings")); + + } + + + TEST_F(ErrorTraceTest, scopedImport) { + } + + + TEST_F(ErrorTraceTest, import) { + } + + + TEST_F(ErrorTraceTest, typeOf) { + } + + + TEST_F(ErrorTraceTest, isNull) { + } + + + TEST_F(ErrorTraceTest, isFunction) { + } + + + TEST_F(ErrorTraceTest, isInt) { + } + + + TEST_F(ErrorTraceTest, isFloat) { + } + + + TEST_F(ErrorTraceTest, isString) { + } + + + TEST_F(ErrorTraceTest, isBool) { + } + + + TEST_F(ErrorTraceTest, isPath) { + } + + + TEST_F(ErrorTraceTest, break) { + } + + + TEST_F(ErrorTraceTest, abort) { + } + + + TEST_F(ErrorTraceTest, throw) { + } + + + TEST_F(ErrorTraceTest, addErrorContext) { + } + + + TEST_F(ErrorTraceTest, ceil) { + ASSERT_TRACE2("ceil \"foo\"", + TypeError, + hintfmt("value is %s while a float was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.ceil")); + + } + + + TEST_F(ErrorTraceTest, floor) { + ASSERT_TRACE2("floor \"foo\"", + TypeError, + hintfmt("value is %s while a float was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.floor")); + + } + + + TEST_F(ErrorTraceTest, tryEval) { + } + + + TEST_F(ErrorTraceTest, getEnv) { + ASSERT_TRACE2("getEnv [ ]", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.getEnv")); + + } + + + TEST_F(ErrorTraceTest, seq) { + } + + + TEST_F(ErrorTraceTest, deepSeq) { + } + + + TEST_F(ErrorTraceTest, trace) { + } + + + TEST_F(ErrorTraceTest, placeholder) { + ASSERT_TRACE2("placeholder []", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.placeholder")); + + } + + + TEST_F(ErrorTraceTest, toPath) { + ASSERT_TRACE2("toPath []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while evaluating the first argument passed to builtins.toPath")); + + ASSERT_TRACE2("toPath \"foo\"", + EvalError, + hintfmt("string '%s' doesn't represent an absolute path", "foo"), + hintfmt("while evaluating the first argument passed to builtins.toPath")); + + } + + + TEST_F(ErrorTraceTest, storePath) { + ASSERT_TRACE2("storePath true", + TypeError, + hintfmt("cannot coerce %s to a string", "a Boolean"), + hintfmt("while evaluating the first argument passed to builtins.storePath")); + + } + + + TEST_F(ErrorTraceTest, pathExists) { + ASSERT_TRACE2("pathExists []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while realising the context of a path")); + + ASSERT_TRACE2("pathExists \"zorglub\"", + EvalError, + hintfmt("string '%s' doesn't represent an absolute path", "zorglub"), + hintfmt("while realising the context of a path")); + + } + + + TEST_F(ErrorTraceTest, baseNameOf) { + ASSERT_TRACE2("baseNameOf []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); + + } + + + TEST_F(ErrorTraceTest, dirOf) { + } + + + TEST_F(ErrorTraceTest, readFile) { + } + + + TEST_F(ErrorTraceTest, findFile) { + } + + + TEST_F(ErrorTraceTest, hashFile) { + } + + + TEST_F(ErrorTraceTest, readDir) { + } + + + TEST_F(ErrorTraceTest, toXML) { + } + + + TEST_F(ErrorTraceTest, toJSON) { + } + + + TEST_F(ErrorTraceTest, fromJSON) { + } + + + TEST_F(ErrorTraceTest, toFile) { + } + + + TEST_F(ErrorTraceTest, filterSource) { + ASSERT_TRACE2("filterSource [] []", + TypeError, + hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + + ASSERT_TRACE2("filterSource [] \"foo\"", + EvalError, + hintfmt("string '%s' doesn't represent an absolute path", "foo"), + hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + + ASSERT_TRACE2("filterSource [] ./.", + TypeError, + hintfmt("value is %s while a function was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.filterSource")); + + // Usupported by store "dummy" + + // ASSERT_TRACE2("filterSource (_: 1) ./.", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "an integer"), + // hintfmt("while adding path '/home/layus/projects/nix'")); + + // ASSERT_TRACE2("filterSource (_: _: 1) ./.", + // TypeError, + // hintfmt("value is %s while a Boolean was expected", "an integer"), + // hintfmt("while evaluating the return value of the path filter function")); + + } + + + TEST_F(ErrorTraceTest, path) { + } + + + TEST_F(ErrorTraceTest, attrNames) { + ASSERT_TRACE2("attrNames []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the argument passed to builtins.attrNames")); + + } + + + TEST_F(ErrorTraceTest, attrValues) { + ASSERT_TRACE2("attrValues []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the argument passed to builtins.attrValues")); + + } + + + TEST_F(ErrorTraceTest, getAttr) { + ASSERT_TRACE2("getAttr [] []", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.getAttr")); + + ASSERT_TRACE2("getAttr \"foo\" []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.getAttr")); + + ASSERT_TRACE2("getAttr \"foo\" {}", + TypeError, + hintfmt("attribute '%s' missing", "foo"), + hintfmt("in the attribute set under consideration")); + + } + + + TEST_F(ErrorTraceTest, unsafeGetAttrPos) { + } + + + TEST_F(ErrorTraceTest, hasAttr) { + ASSERT_TRACE2("hasAttr [] []", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.hasAttr")); + + ASSERT_TRACE2("hasAttr \"foo\" []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.hasAttr")); + + } + + + TEST_F(ErrorTraceTest, isAttrs) { + } + + + TEST_F(ErrorTraceTest, removeAttrs) { + ASSERT_TRACE2("removeAttrs \"\" \"\"", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + + ASSERT_TRACE2("removeAttrs \"\" [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + + ASSERT_TRACE2("removeAttrs \"\" [ \"1\" ]", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + + } + + + TEST_F(ErrorTraceTest, listToAttrs) { + ASSERT_TRACE2("listToAttrs 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the argument passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating an element of the list passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ {} ]", + TypeError, + hintfmt("attribute '%s' missing", "name"), + hintfmt("in a {name=...; value=...;} pair")); + + ASSERT_TRACE2("listToAttrs [ { name = 1; } ]", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); + + ASSERT_TRACE2("listToAttrs [ { name = \"foo\"; } ]", + TypeError, + hintfmt("attribute '%s' missing", "value"), + hintfmt("in a {name=...; value=...;} pair")); + + } + + + TEST_F(ErrorTraceTest, intersectAttrs) { + ASSERT_TRACE2("intersectAttrs [] []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.intersectAttrs")); + + ASSERT_TRACE2("intersectAttrs {} []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.intersectAttrs")); + + } + + + TEST_F(ErrorTraceTest, catAttrs) { + ASSERT_TRACE2("catAttrs [] {}", + TypeError, + hintfmt("value is %s while a string was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" {}", + TypeError, + hintfmt("value is %s while a list was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + + ASSERT_TRACE2("catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + + } + + + TEST_F(ErrorTraceTest, functionArgs) { + ASSERT_TRACE1("functionArgs {}", + TypeError, + hintfmt("'functionArgs' requires a function")); + + } + + + TEST_F(ErrorTraceTest, mapAttrs) { + ASSERT_TRACE2("mapAttrs [] []", + TypeError, + hintfmt("value is %s while a set was expected", "a list"), + hintfmt("while evaluating the second argument passed to builtins.mapAttrs")); + + // XXX: defered + // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "a string"), + // hintfmt("while evaluating the attribute 'foo'")); + + // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "a string"), + // hintfmt("while evaluating the attribute 'foo'")); + + // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", + // TypeError, + // hintfmt("cannot coerce %s to a string", "an integer"), + // hintfmt("while evaluating a path segment")); + + } + + + TEST_F(ErrorTraceTest, zipAttrsWith) { + ASSERT_TRACE2("zipAttrsWith [] [ 1 ]", + TypeError, + hintfmt("value is %s while a function was expected", "a list"), + hintfmt("while evaluating the first argument passed to builtins.zipAttrsWith")); + + ASSERT_TRACE2("zipAttrsWith (_: 1) [ 1 ]", + TypeError, + hintfmt("value is %s while a set was expected", "an integer"), + hintfmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); + + // XXX: How to properly tell that the fucntion takes two arguments ? + // The same question also applies to sort, and maybe others. + // Due to lazyness, we only create a thunk, and it fails later on. + // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", + // TypeError, + // hintfmt("attempt to call something which is not a function but %s", "an integer"), + // hintfmt("while evaluating the attribute 'foo'")); + + // XXX: Also deferred deeply + // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", + // TypeError, + // hintfmt("cannot coerce %s to a string", "a list"), + // hintfmt("while evaluating a path segment")); + + } + + + TEST_F(ErrorTraceTest, isList) { + } + + + TEST_F(ErrorTraceTest, elemAt) { + ASSERT_TRACE2("elemAt \"foo\" (-1)", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.elemAt")); + + ASSERT_TRACE1("elemAt [] (-1)", + Error, + hintfmt("list index %d is out of bounds", -1)); + + ASSERT_TRACE1("elemAt [\"foo\"] 3", + Error, + hintfmt("list index %d is out of bounds", 3)); + + } + + + TEST_F(ErrorTraceTest, head) { + ASSERT_TRACE2("head 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.elemAt")); + + ASSERT_TRACE1("head []", + Error, + hintfmt("list index %d is out of bounds", 0)); + + } + + + TEST_F(ErrorTraceTest, tail) { + ASSERT_TRACE2("tail 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.tail")); + + ASSERT_TRACE1("tail []", + Error, + hintfmt("'tail' called on an empty list")); + + } + + + TEST_F(ErrorTraceTest, map) { + ASSERT_TRACE2("map 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.map")); + + ASSERT_TRACE2("map 1 [ 1 ]", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.map")); + + } + + + TEST_F(ErrorTraceTest, filter) { + ASSERT_TRACE2("filter 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.filter")); + + ASSERT_TRACE2("filter 1 [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.filter")); + + ASSERT_TRACE2("filter (_: 5) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the filtering function passed to builtins.filter")); + + } + + + TEST_F(ErrorTraceTest, elem) { + ASSERT_TRACE2("elem 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.elem")); + + } + + + TEST_F(ErrorTraceTest, concatLists) { + ASSERT_TRACE2("concatLists 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.concatLists")); + + ASSERT_TRACE2("concatLists [ 1 ]", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + + ASSERT_TRACE2("concatLists [ [1] \"foo\" ]", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + + } + + + TEST_F(ErrorTraceTest, length) { + ASSERT_TRACE2("length 1", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.length")); + + ASSERT_TRACE2("length \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the first argument passed to builtins.length")); + + } + + + TEST_F(ErrorTraceTest, foldlPrime) { + ASSERT_TRACE2("foldl' 1 \"foo\" true", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.foldlStrict")); + + ASSERT_TRACE2("foldl' (_: 1) \"foo\" true", + TypeError, + hintfmt("value is %s while a list was expected", "a Boolean"), + hintfmt("while evaluating the third argument passed to builtins.foldlStrict")); + + ASSERT_TRACE1("foldl' (_: 1) \"foo\" [ true ]", + TypeError, + hintfmt("attempt to call something which is not a function but %s", "an integer")); + + ASSERT_TRACE2("foldl' (a: b: a && b) \"foo\" [ true ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("in the left operand of the AND (&&) operator")); + + } + + + TEST_F(ErrorTraceTest, any) { + ASSERT_TRACE2("any 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.any")); + + ASSERT_TRACE2("any (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.any")); + + ASSERT_TRACE2("any (_: 1) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the function passed to builtins.any")); + + } + + + TEST_F(ErrorTraceTest, all) { + ASSERT_TRACE2("all 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.all")); + + ASSERT_TRACE2("all (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.all")); + + ASSERT_TRACE2("all (_: 1) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the function passed to builtins.all")); + + } + + + TEST_F(ErrorTraceTest, genList) { + ASSERT_TRACE2("genList 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.genList")); + + ASSERT_TRACE2("genList 1 2", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.genList", "an integer")); + + // XXX: defered + // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", + // TypeError, + // hintfmt("cannot add %s to an integer", "a string"), + // hintfmt("while evaluating anonymous lambda")); + + ASSERT_TRACE1("genList false (-3)", + EvalError, + hintfmt("cannot create list of size %d", -3)); + + } + + + TEST_F(ErrorTraceTest, sort) { + ASSERT_TRACE2("sort 1 \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.sort")); + + ASSERT_TRACE2("sort 1 [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.sort")); + + ASSERT_TRACE1("sort (_: 1) [ \"foo\" \"bar\" ]", + TypeError, + hintfmt("attempt to call something which is not a function but %s", "an integer")); + + ASSERT_TRACE2("sort (_: _: 1) [ \"foo\" \"bar\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the sorting function passed to builtins.sort")); + + // XXX: Trace too deep, need better asserts + // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", + // TypeError, + // hintfmt("cannot compare %s with %s", "a string", "a set")); + + // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", + // TypeError, + // hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + + } + + + TEST_F(ErrorTraceTest, partition) { + ASSERT_TRACE2("partition 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.partition")); + + ASSERT_TRACE2("partition (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.partition")); + + ASSERT_TRACE2("partition (_: 1) [ \"foo\" ]", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the return value of the partition function passed to builtins.partition")); + + } + + + TEST_F(ErrorTraceTest, groupBy) { + ASSERT_TRACE2("groupBy 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.groupBy")); + + ASSERT_TRACE2("groupBy (_: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.groupBy")); + + ASSERT_TRACE2("groupBy (x: x) [ \"foo\" \"bar\" 1 ]", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); + + } + + + TEST_F(ErrorTraceTest, concatMap) { + ASSERT_TRACE2("concatMap 1 \"foo\"", + TypeError, + hintfmt("value is %s while a function was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: 1) \"foo\"", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the second argument passed to builtins.concatMap")); + + ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", + TypeError, + hintfmt("value is %s while a list was expected", "an integer"), + hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + + ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + + } + + + TEST_F(ErrorTraceTest, add) { + ASSERT_TRACE2("add \"foo\" 1", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first argument of the addition")); + + ASSERT_TRACE2("add 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument of the addition")); + + } + + + TEST_F(ErrorTraceTest, sub) { + ASSERT_TRACE2("sub \"foo\" 1", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first argument of the subtraction")); + + ASSERT_TRACE2("sub 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument of the subtraction")); + + } + + + TEST_F(ErrorTraceTest, mul) { + ASSERT_TRACE2("mul \"foo\" 1", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first argument of the multiplication")); + + ASSERT_TRACE2("mul 1 \"foo\"", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument of the multiplication")); + + } + + + TEST_F(ErrorTraceTest, div) { + ASSERT_TRACE2("div \"foo\" 1 # TODO: an integer was expected -> a number", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the first operand of the division")); + + ASSERT_TRACE2("div 1 \"foo\"", + TypeError, + hintfmt("value is %s while a float was expected", "a string"), + hintfmt("while evaluating the second operand of the division")); + + ASSERT_TRACE1("div \"foo\" 0", + EvalError, + hintfmt("division by zero")); + + } + + + TEST_F(ErrorTraceTest, bitAnd) { + ASSERT_TRACE2("bitAnd 1.1 2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the first argument passed to builtins.bitAnd")); + + ASSERT_TRACE2("bitAnd 1 2.2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the second argument passed to builtins.bitAnd")); + + } + + + TEST_F(ErrorTraceTest, bitOr) { + ASSERT_TRACE2("bitOr 1.1 2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the first argument passed to builtins.bitOr")); + + ASSERT_TRACE2("bitOr 1 2.2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the second argument passed to builtins.bitOr")); + + } + + + TEST_F(ErrorTraceTest, bitXor) { + ASSERT_TRACE2("bitXor 1.1 2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the first argument passed to builtins.bitXor")); + + ASSERT_TRACE2("bitXor 1 2.2", + TypeError, + hintfmt("value is %s while an integer was expected", "a float"), + hintfmt("while evaluating the second argument passed to builtins.bitXor")); + + } + + + TEST_F(ErrorTraceTest, lessThan) { + ASSERT_TRACE1("lessThan 1 \"foo\"", + EvalError, + hintfmt("cannot compare %s with %s", "an integer", "a string")); + + ASSERT_TRACE1("lessThan {} {}", + EvalError, + hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + + ASSERT_TRACE2("lessThan [ 1 2 ] [ \"foo\" ]", + EvalError, + hintfmt("cannot compare %s with %s", "an integer", "a string"), + hintfmt("while comparing two list elements")); + + } + + + TEST_F(ErrorTraceTest, toString) { + ASSERT_TRACE2("toString { a = 1; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the first argument passed to builtins.toString")); + + } + + + TEST_F(ErrorTraceTest, substring) { + ASSERT_TRACE2("substring {} \"foo\" true", + TypeError, + hintfmt("value is %s while an integer was expected", "a set"), + hintfmt("while evaluating the first argument (the start offset) passed to builtins.substring")); + + ASSERT_TRACE2("substring 3 \"foo\" true", + TypeError, + hintfmt("value is %s while an integer was expected", "a string"), + hintfmt("while evaluating the second argument (the substring length) passed to builtins.substring")); + + ASSERT_TRACE2("substring 0 3 {}", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); + + ASSERT_TRACE1("substring (-3) 3 \"sometext\"", + EvalError, + hintfmt("negative start position in 'substring'")); + + } + + + TEST_F(ErrorTraceTest, stringLength) { + ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the argument passed to builtins.stringLength")); + + } + + + TEST_F(ErrorTraceTest, hashString) { + ASSERT_TRACE2("hashString 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.hashString")); + + ASSERT_TRACE1("hashString \"foo\" \"content\"", + UsageError, + hintfmt("unknown hash algorithm '%s'", "foo")); + + ASSERT_TRACE2("hashString \"sha256\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.hashString")); + + } + + + TEST_F(ErrorTraceTest, match) { + ASSERT_TRACE2("match 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.match")); + + ASSERT_TRACE2("match \"foo\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.match")); + + ASSERT_TRACE1("match \"(.*\" \"\"", + EvalError, + hintfmt("invalid regular expression '%s'", "(.*")); + + } + + + TEST_F(ErrorTraceTest, split) { + ASSERT_TRACE2("split 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.split")); + + ASSERT_TRACE2("split \"foo\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.split")); + + ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", + EvalError, + hintfmt("invalid regular expression '%s'", "f(o*o")); + + } + + + TEST_F(ErrorTraceTest, concatStringsSep) { + ASSERT_TRACE2("concatStringsSep 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); + + ASSERT_TRACE2("concatStringsSep \"foo\" {}", + TypeError, + hintfmt("value is %s while a list was expected", "a set"), + hintfmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); + + ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", + TypeError, + hintfmt("cannot coerce %s to a string", "an integer"), + hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); + + } + + + TEST_F(ErrorTraceTest, parseDrvName) { + ASSERT_TRACE2("parseDrvName 1", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.parseDrvName")); + + } + + + TEST_F(ErrorTraceTest, compareVersions) { + ASSERT_TRACE2("compareVersions 1 {}", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.compareVersions")); + + ASSERT_TRACE2("compareVersions \"abd\" {}", + TypeError, + hintfmt("value is %s while a string was expected", "a set"), + hintfmt("while evaluating the second argument passed to builtins.compareVersions")); + + } + + + TEST_F(ErrorTraceTest, splitVersion) { + ASSERT_TRACE2("splitVersion 1", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the first argument passed to builtins.splitVersion")); + + } + + + TEST_F(ErrorTraceTest, traceVerbose) { + } + + + /* // Needs different ASSERTs + TEST_F(ErrorTraceTest, derivationStrict) { + ASSERT_TRACE2("derivationStrict \"\"", + TypeError, + hintfmt("value is %s while a set was expected", "a string"), + hintfmt("while evaluating the argument passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict {}", + TypeError, + hintfmt("attribute '%s' missing", "name"), + hintfmt("in the attrset passed as argument to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = 1; }", + TypeError, + hintfmt("value is %s while a string was expected", "an integer"), + hintfmt("while evaluating the `name` attribute passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; }", + TypeError, + hintfmt("required attribute 'builder' missing"), + hintfmt("while evaluating derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "an integer"), + hintfmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", + TypeError, + hintfmt("invalid value '15' for 'outputHashMode' attribute"), + hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", + TypeError, + hintfmt("invalid value 'custom' for 'outputHashMode' attribute"), + hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", + TypeError, + hintfmt("invalid derivation output name 'drv'"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", + TypeError, + hintfmt("derivation cannot have an empty set of outputs"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drv\" ]; }", + TypeError, + hintfmt("invalid derivation output name 'drv'"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", + TypeError, + hintfmt("duplicate derivation output 'out'"), + hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("while evaluating the attribute '__contentAddressed' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", + TypeError, + hintfmt("value is %s while a Boolean was expected", "a string"), + hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", + TypeError, + hintfmt("value is %s while a list was expected", "a string"), + hintfmt("while evaluating the attribute 'args' of derivation 'foo'")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating an element of the argument list")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating an element of the argument list")); + + ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", + TypeError, + hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); + + } + */ + +} /* namespace nix */ diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index f1ea1b197..411bc0ac3 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" #include "value-to-json.hh" namespace nix { diff --git a/src/libexpr/tests/libexprtests.hh b/src/libexpr/tests/libexpr.hh index 4f6915882..8534d9567 100644 --- a/src/libexpr/tests/libexprtests.hh +++ b/src/libexpr/tests/libexpr.hh @@ -7,17 +7,19 @@ #include "eval-inline.hh" #include "store-api.hh" +#include "tests/libstore.hh" namespace nix { - class LibExprTest : public ::testing::Test { + class LibExprTest : public LibStoreTest { public: static void SetUpTestSuite() { + LibStoreTest::SetUpTestSuite(); initGC(); } protected: LibExprTest() - : store(openStore("dummy://")) + : LibStoreTest() , state({}, store) { } @@ -35,7 +37,6 @@ namespace nix { return state.symbols.create(value); } - ref<Store> store; EvalState state; }; @@ -123,7 +124,7 @@ namespace nix { MATCHER_P(IsAttrsOfSize, n, fmt("Is a set of size [%1%]", n)) { if (arg.type() != nAttrs) { - *result_listener << "Expexted set got " << arg.type(); + *result_listener << "Expected set got " << arg.type(); return false; } else if (arg.attrs->size() != (size_t)n) { *result_listener << "Expected a set with " << n << " attributes but got " << arg.attrs->size(); diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index b95980cab..3e5504f71 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -2,14 +2,18 @@ check: libexpr-tests_RUN programs += libexpr-tests +libexpr-tests_NAME := libnixexpr-tests + libexpr-tests_DIR := $(d) libexpr-tests_INSTALL_DIR := -libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libexpr libutil libstore libfetchers +libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index 16cf66d2c..e1d3ac503 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -1,7 +1,7 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { class CaptureLogger : public Logger @@ -151,20 +151,7 @@ namespace nix { // The `y` attribute is at position const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }"; auto v = eval(expr); - ASSERT_THAT(v, IsAttrsOfSize(3)); - - auto file = v.attrs->find(createSymbol("file")); - ASSERT_NE(file, nullptr); - // FIXME: The file when running these tests is the input string?!? - ASSERT_THAT(*file->value, IsStringEq(expr)); - - auto line = v.attrs->find(createSymbol("line")); - ASSERT_NE(line, nullptr); - ASSERT_THAT(*line->value, IsIntEq(1)); - - auto column = v.attrs->find(createSymbol("column")); - ASSERT_NE(column, nullptr); - ASSERT_THAT(*column->value, IsIntEq(33)); + ASSERT_THAT(v, IsNull()); } TEST_F(PrimOpTest, hasAttr) { @@ -617,7 +604,7 @@ namespace nix { TEST_F(PrimOpTest, storeDir) { auto v = eval("builtins.storeDir"); - ASSERT_THAT(v, IsStringEq("/nix/store")); + ASSERT_THAT(v, IsStringEq(settings.nixStore)); } TEST_F(PrimOpTest, nixVersion) { @@ -836,4 +823,10 @@ namespace nix { for (const auto [n, elem] : enumerate(v.listItems())) ASSERT_THAT(*elem, IsStringEq(expected[n])); } + + TEST_F(PrimOpTest, genericClosure_not_strict) { + // Operator should not be used when startSet is empty + auto v = eval("builtins.genericClosure { startSet = []; }"); + ASSERT_THAT(v, IsListOfSize(0)); + } } /* namespace nix */ diff --git a/src/libexpr/tests/trivial.cc b/src/libexpr/tests/trivial.cc index 8ce276e52..171727ac7 100644 --- a/src/libexpr/tests/trivial.cc +++ b/src/libexpr/tests/trivial.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { // Testing of trivial expressions diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc new file mode 100644 index 000000000..083359b7a --- /dev/null +++ b/src/libexpr/tests/value/context.cc @@ -0,0 +1,127 @@ +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include "tests/path.hh" +#include "tests/libexpr.hh" +#include "tests/value/context.hh" + +namespace nix { + +// Testing of trivial expressions +struct NixStringContextElemTest : public LibExprTest { + const Store & store() const { + return *LibExprTest::store; + } +}; + +TEST_F(NixStringContextElemTest, empty_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), ""), + BadNixStringContextElem); +} + +TEST_F(NixStringContextElemTest, single_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "!"), + BadNixStringContextElem); +} + +TEST_F(NixStringContextElemTest, double_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "!!/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, eq_slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "=/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(store(), opaque); + auto * p = std::get_if<NixStringContextElem::Opaque>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store().parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(store()), opaque); +} + +TEST_F(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(store(), drvDeep); + auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); + ASSERT_EQ(elem.to_string(store()), drvDeep); +} + +TEST_F(NixStringContextElemTest, built) { + std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(store(), built); + auto * p = std::get_if<NixStringContextElem::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); + ASSERT_EQ(elem.to_string(store()), built); +} + +} + +namespace rc { +using namespace nix; + +Gen<NixStringContextElem::Opaque> Arbitrary<NixStringContextElem::Opaque>::arbitrary() +{ + return gen::just(NixStringContextElem::Opaque { + .path = *gen::arbitrary<StorePath>(), + }); +} + +Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary<StorePath>(), + }); +} + +Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitrary() +{ + return gen::just(NixStringContextElem::Built { + .drvPath = *gen::arbitrary<StorePath>(), + .output = (*gen::arbitrary<StorePathName>()).name, + }); +} + +Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 2)) { + case 0: + return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>()); + case 1: + return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>()); + default: + return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>()); + } +} + +} + +namespace nix { + +RC_GTEST_FIXTURE_PROP( + NixStringContextElemTest, + prop_round_rip, + (const NixStringContextElem & o)) +{ + RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); +} + +} diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh new file mode 100644 index 000000000..54d21760e --- /dev/null +++ b/src/libexpr/tests/value/context.hh @@ -0,0 +1,30 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <value/context.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<NixStringContextElem::Opaque> { + static Gen<NixStringContextElem::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<NixStringContextElem::Built> { + static Gen<NixStringContextElem::Built> arbitrary(); +}; + +template<> +struct Arbitrary<NixStringContextElem::DrvDeep> { + static Gen<NixStringContextElem::DrvDeep> arbitrary(); +}; + +template<> +struct Arbitrary<NixStringContextElem> { + static Gen<NixStringContextElem> arbitrary(); +}; + +} diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5dc453b2e..c35c876e3 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" #include "util.hh" +#include "store-api.hh" #include <cstdlib> #include <iomanip> @@ -35,7 +36,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) - out = state.copyPathToStore(context, v.path); + out = state.store->printStorePath(state.copyPathToStore(context, v.path)); else out = v.path; break; diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 7c3bf9492..3f6222768 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -24,7 +24,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) { - xmlAttrs["path"] = pos.file; + if (auto path = std::get_if<Path>(&pos.origin)) + xmlAttrs["path"] = *path; xmlAttrs["line"] = (format("%1%") % pos.line).str(); xmlAttrs["column"] = (format("%1%") % pos.column).str(); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 5adac72f8..508dbe218 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include <cassert> #include "symbol-table.hh" +#include "value/context.hh" #if HAVE_BOEHMGC #include <gc/gc_allocator.h> @@ -67,8 +68,6 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -typedef std::pair<StorePath, std::string> NixStringContextElem; -typedef std::vector<NixStringContextElem> NixStringContext; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc new file mode 100644 index 000000000..61d9c53df --- /dev/null +++ b/src/libexpr/value/context.cc @@ -0,0 +1,67 @@ +#include "value/context.hh" +#include "store-api.hh" + +#include <optional> + +namespace nix { + +NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +{ + std::string_view s = s0; + + if (s.size() == 0) { + throw BadNixStringContextElem(s0, + "String context element should never be an empty string"); + } + switch (s.at(0)) { + case '!': { + s = s.substr(1); // advance string to parse after first ! + size_t index = s.find("!"); + // This makes index + 1 safe. Index can be the length (one after index + // of last character), so given any valid character index --- a + // successful find --- we can add one. + if (index == std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element beginning with '!' should have a second '!'"); + } + return NixStringContextElem::Built { + .drvPath = store.parseStorePath(s.substr(index + 1)), + .output = std::string(s.substr(0, index)), + }; + } + case '=': { + return NixStringContextElem::DrvDeep { + .drvPath = store.parseStorePath(s.substr(1)), + }; + } + default: { + return NixStringContextElem::Opaque { + .path = store.parseStorePath(s), + }; + } + } +} + +std::string NixStringContextElem::to_string(const Store & store) const { + return std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + std::string res; + res += '!'; + res += b.output; + res += '!'; + res += store.printStorePath(b.drvPath); + return res; + }, + [&](const NixStringContextElem::DrvDeep & d) { + std::string res; + res += '='; + res += store.printStorePath(d.drvPath); + return res; + }, + [&](const NixStringContextElem::Opaque & o) { + return store.printStorePath(o.path); + }, + }, raw()); +} + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh new file mode 100644 index 000000000..721563cba --- /dev/null +++ b/src/libexpr/value/context.hh @@ -0,0 +1,97 @@ +#pragma once + +#include "util.hh" +#include "comparator.hh" +#include "path.hh" + +#include <variant> + +#include <nlohmann/json_fwd.hpp> + +namespace nix { + +class BadNixStringContextElem : public Error +{ +public: + std::string_view raw; + + template<typename... Args> + BadNixStringContextElem(std::string_view raw_, const Args & ... args) + : Error("") + { + raw = raw_; + auto hf = hintfmt(args...); + err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw); + } +}; + +class Store; + +/* Plain opaque path to some store object. + + Encoded as just the path: ‘<path>’. +*/ +struct NixStringContextElem_Opaque { + StorePath path; + + GENERATE_CMP(NixStringContextElem_Opaque, me->path); +}; + +/* Path to a derivation and its entire build closure. + + The path doesn't just refer to derivation itself and its closure, but + also all outputs of all derivations in that closure (including the + root derivation). + + Encoded in the form ‘=<drvPath>’. +*/ +struct NixStringContextElem_DrvDeep { + StorePath drvPath; + + GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); +}; + +/* Derivation output. + + Encoded in the form ‘!<output>!<drvPath>’. +*/ +struct NixStringContextElem_Built { + StorePath drvPath; + std::string output; + + GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); +}; + +using _NixStringContextElem_Raw = std::variant< + NixStringContextElem_Opaque, + NixStringContextElem_DrvDeep, + NixStringContextElem_Built +>; + +struct NixStringContextElem : _NixStringContextElem_Raw { + using Raw = _NixStringContextElem_Raw; + using Raw::Raw; + + using Opaque = NixStringContextElem_Opaque; + using DrvDeep = NixStringContextElem_DrvDeep; + using Built = NixStringContextElem_Built; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + inline Raw & raw() { + return static_cast<Raw &>(*this); + } + + /* Decode a context string, one of: + - ‘<path>’ + - ‘=<path>’ + - ‘!<name>!<path>’ + */ + static NixStringContextElem parse(const Store & store, std::string_view s); + std::string to_string(const Store & store) const; +}; + +typedef std::vector<NixStringContextElem> NixStringContext; + +} diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6452143a1..f33cbdcfc 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -71,7 +71,12 @@ struct FetchSettings : public Config "Whether to warn about dirty Git/Mercurial trees."}; Setting<std::string> flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - "Path or URI of the global flake registry."}; + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )"}; + Setting<bool> useRegistries{this, true, "use-registries", "Whether to use flake registries to resolve flake references."}; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6957d2da4..c767e72e5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -266,7 +266,7 @@ std::optional<time_t> Input::getLastModified() const return {}; } -ParsedURL InputScheme::toURL(const Input & input) +ParsedURL InputScheme::toURL(const Input & input) const { throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } @@ -274,7 +274,7 @@ ParsedURL InputScheme::toURL(const Input & input) Input InputScheme::applyOverrides( const Input & input, std::optional<std::string> ref, - std::optional<Hash> rev) + std::optional<Hash> rev) const { if (ref) throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); @@ -293,7 +293,7 @@ void InputScheme::markChangedFile(const Input & input, std::string_view file, st assert(false); } -void InputScheme::clone(const Input & input, const Path & destDir) +void InputScheme::clone(const Input & input, const Path & destDir) const { throw Error("do not know how to clone input '%s'", input.to_string()); } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index bc9a76b0b..17da37f47 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -107,26 +107,25 @@ public: * recognized. The Input object contains the information the fetcher * needs to actually perform the "fetch()" when called. */ - struct InputScheme { virtual ~InputScheme() { } - virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0; + virtual std::optional<Input> inputFromURL(const ParsedURL & url) const = 0; - virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0; + virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0; - virtual ParsedURL toURL(const Input & input); + virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) = 0; + virtual bool hasAllInfo(const Input & input) const = 0; virtual Input applyOverrides( const Input & input, std::optional<std::string> ref, - std::optional<Hash> rev); + std::optional<Hash> rev) const; - virtual void clone(const Input & input, const Path & destDir); + virtual void clone(const Input & input, const Path & destDir) const; virtual std::optional<Path> getSourcePath(const Input & input); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7b7a1be35..309a143f5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -18,6 +18,7 @@ using namespace std::string_literals; namespace nix::fetchers { + namespace { // Explicit initial branch of our bare repo to suppress warnings from new version of git. @@ -26,23 +27,23 @@ namespace { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -bool isCacheFileWithinTtl(const time_t now, const struct stat & st) +bool isCacheFileWithinTtl(time_t now, const struct stat & st) { return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path& path, const time_t& touch_time) +bool touchCacheFile(const Path & path, time_t touch_time) { - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; + struct timeval times[2]; + times[0].tv_sec = touch_time; + times[0].tv_usec = 0; + times[1].tv_sec = touch_time; + times[1].tv_usec = 0; - return lutimes(path.c_str(), times) == 0; + return lutimes(path.c_str(), times) == 0; } -Path getCachePath(std::string key) +Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, key).to_string(Base32, false); @@ -57,13 +58,12 @@ Path getCachePath(std::string key) // ... std::optional<std::string> readHead(const Path & path) { - auto [exit_code, output] = runProgram(RunOptions { + auto [status, output] = runProgram(RunOptions { .program = "git", + // FIXME: use 'HEAD' to avoid returning all refs .args = {"ls-remote", "--symref", path}, }); - if (exit_code != 0) { - return std::nullopt; - } + if (status != 0) return std::nullopt; std::string_view line = output; line = line.substr(0, line.find("\n")); @@ -82,12 +82,11 @@ std::optional<std::string> readHead(const Path & path) } // Persist the HEAD ref from the remote repo in the local cached repo. -bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) +bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { Path cacheDir = getCachePath(actualUrl); - auto gitDir = "."; try { - runProgram("git", true, { "-C", cacheDir, "--git-dir", gitDir, "symbolic-ref", "--", "HEAD", headRef }); + runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { if (!WIFEXITED(e.status)) throw; return false; @@ -96,7 +95,7 @@ bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) return true; } -std::optional<std::string> readHeadCached(const std::string& actualUrl) +std::optional<std::string> readHeadCached(const std::string & actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. @@ -110,16 +109,15 @@ std::optional<std::string> readHeadCached(const std::string& actualUrl) cachedRef = readHead(cacheDir); if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && - isCacheFileWithinTtl(now, st)) { + isCacheFileWithinTtl(now, st)) + { debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); return cachedRef; } } auto ref = readHead(actualUrl); - if (ref) { - return ref; - } + if (ref) return ref; if (cachedRef) { // If the cached git ref is expired in fetch() below, and the 'git fetch' @@ -250,7 +248,7 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co struct GitInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -265,7 +263,7 @@ struct GitInputScheme : InputScheme Attrs attrs; attrs.emplace("type", "git"); - for (auto &[name, value] : url.query) { + for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); else if (name == "shallow" || name == "submodules") @@ -279,7 +277,7 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; @@ -302,7 +300,7 @@ struct GitInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme != "git") url.scheme = "git+" + url.scheme; @@ -313,7 +311,7 @@ struct GitInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { bool maybeDirty = !input.getRef(); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); @@ -325,7 +323,7 @@ struct GitInputScheme : InputScheme Input applyOverrides( const Input & input, std::optional<std::string> ref, - std::optional<Hash> rev) override + std::optional<Hash> rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); @@ -335,7 +333,7 @@ struct GitInputScheme : InputScheme return res; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto [isLocal, actualUrl] = getActualUrl(input); @@ -603,9 +601,9 @@ struct GitInputScheme : InputScheme { throw Error( "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", input.getRev()->gitRev(), *input.getRef(), actualUrl @@ -617,15 +615,42 @@ struct GitInputScheme : InputScheme AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + { + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + } runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + /* Ensure that we use the correct origin for fetching + submodules. This matters for submodules with relative + URLs. */ + if (isLocal) { + writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + + /* Restore the config.bare setting we may have just + copied erroneously from the user's repo. */ + runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); + } else + runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + + /* As an optimisation, copy the modules directory of the + source repo if it exists. */ + auto modulesPath = repoDir + "/" + gitDir + "/modules"; + if (pathExists(modulesPath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); + } + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + } filter = isNotDotGitDirectory; } else { diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2115ce2f5..1ed09d30d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -26,11 +26,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() = 0; + virtual std::string type() const = 0; virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0; - std::optional<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) const override { if (url.scheme != type()) return {}; @@ -100,7 +100,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != type()) return {}; @@ -116,7 +116,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); @@ -132,7 +132,7 @@ struct GitArchiveInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } @@ -140,7 +140,7 @@ struct GitArchiveInputScheme : InputScheme Input applyOverrides( const Input & _input, std::optional<std::string> ref, - std::optional<Hash> rev) override + std::optional<Hash> rev) const override { auto input(_input); if (rev && ref) @@ -227,7 +227,7 @@ struct GitArchiveInputScheme : InputScheme struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() override { return "github"; } + std::string type() const override { return "github"; } std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override { @@ -240,14 +240,29 @@ struct GitHubInputScheme : GitArchiveInputScheme return std::pair<std::string, std::string>("Authorization", fmt("token %s", token)); } + std::string getHost(const Input & input) const + { + return maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + } + + std::string getOwner(const Input & input) const + { + return getStrAttr(input.attrs, "owner"); + } + + std::string getRepo(const Input & input) const + { + return getStrAttr(input.attrs, "repo"); + } + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); auto url = fmt( host == "github.com" ? "https://api.%s/repos/%s/%s/commits/%s" : "https://%s/api/v3/repos/%s/%s/commits/%s", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getOwner(input), getRepo(input), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -262,8 +277,10 @@ struct GitHubInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); + Headers headers = makeHeadersWithAuthTokens(host); + // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. const auto urlFmt = @@ -273,17 +290,17 @@ struct GitHubInputScheme : GitArchiveInputScheme ? "https://%s/%s/%s/archive/%s.tar.gz" : "https://api.%s/repos/%s/%s/tarball/%s"; - const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->to_string(Base16, false)); return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); Input::fromURL(fmt("git+https://%s/%s/%s.git", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); } @@ -291,7 +308,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() override { return "gitlab"; } + std::string type() const override { return "gitlab"; } std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override { @@ -346,7 +363,7 @@ struct GitLabInputScheme : GitArchiveInputScheme return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere @@ -359,7 +376,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() override { return "sourcehut"; } + std::string type() const override { return "sourcehut"; } std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override { @@ -433,7 +450,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); Input::fromURL(fmt("git+https://%s/%s/%s", diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9288fc6cf..b99504a16 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) const override { if (url.scheme != "flake") return {}; @@ -50,7 +50,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::optional<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; @@ -68,7 +68,7 @@ struct IndirectInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { ParsedURL url; url.scheme = "flake"; @@ -78,7 +78,7 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return false; } @@ -86,7 +86,7 @@ struct IndirectInputScheme : InputScheme Input applyOverrides( const Input & _input, std::optional<std::string> ref, - std::optional<Hash> rev) override + std::optional<Hash> rev) const override { auto input(_input); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5c5671681..86e8f81f4 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional<std::string> struct MercurialInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -69,7 +69,7 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; @@ -89,7 +89,7 @@ struct MercurialInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); url.scheme = "hg+" + url.scheme; @@ -98,7 +98,7 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { // FIXME: ugly, need to distinguish between dirty and clean // default trees. @@ -108,7 +108,7 @@ struct MercurialInputScheme : InputScheme Input applyOverrides( const Input & input, std::optional<std::string> ref, - std::optional<Hash> rev) override + std::optional<Hash> rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f0ef97da5..61541e69d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) const override { if (url.scheme != "path") return {}; @@ -32,7 +32,7 @@ struct PathInputScheme : InputScheme return input; } - std::optional<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "path") return {}; @@ -54,7 +54,7 @@ struct PathInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto query = attrsToQuery(input.attrs); query.erase("path"); @@ -66,7 +66,7 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index acd1ff866..43c03beec 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -153,6 +153,9 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store) { static auto reg = [&]() { auto path = fetchSettings.flakeRegistry.get(); + if (path == "") { + return std::make_shared<Registry>(Registry::Global); // empty registry + } if (!hasPrefix(path, "/")) { auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 6c551bd93..e9686262a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -185,7 +185,7 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url) const = 0; - std::optional<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) const override { if (!isValidURL(url)) return std::nullopt; @@ -203,7 +203,7 @@ struct CurlInputScheme : InputScheme return input; } - std::optional<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) const override { auto type = maybeGetStrAttr(attrs, "type"); if (type != inputType()) return {}; @@ -220,16 +220,17 @@ struct CurlInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip + // files don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 37b03dcd4..fb3ead6fa 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 961f4e18a..e9205a5e5 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -132,7 +132,7 @@ public: log(*state, lvl, fs.s); } - void logEI(const ErrorInfo &ei) override + void logEI(const ErrorInfo & ei) override { auto state(state_.lock()); @@ -180,10 +180,12 @@ public: auto machineName = getS(fields, 1); if (machineName != "") i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); - auto curRound = getI(fields, 2); - auto nrRounds = getI(fields, 3); - if (nrRounds != 1) - i->s += fmt(" (round %d/%d)", curRound, nrRounds); + + // Used to be curRound and nrRounds, but the + // implementation was broken for a long time. + if (getI(fields, 2) != 1 || getI(fields, 3) != 1) { + throw Error("log message indicated repeating builds, but this is not currently implemented"); + } i->name = DrvName(name).name; } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a58428762..d4871a8e2 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -235,6 +235,7 @@ void initNix() #endif preloadNSS(); + initLibStore(); } @@ -362,6 +363,7 @@ void printVersion(const std::string & programName) << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; + std::cout << "Data directory: " << settings.nixDataDir << "\n"; } throw Exit(); } @@ -402,8 +404,6 @@ int handleExceptions(const std::string & programName, std::function<void()> fun) return 1; } catch (BaseError & e) { logError(e.info()); - if (e.hasTrace() && !loggerSettings.showTrace.get()) - printError("(use '--show-trace' to show detailed location information)"); return e.status; } catch (std::bad_alloc & e) { printError(error + "out of memory"); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3c37fd627..1715374a6 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -39,7 +39,6 @@ void printVersion(const std::string & programName); void printGCWarning(); class Store; -struct StorePathWithOutputs; void printMissing( ref<Store> store, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 12d0c32fb..751cf8c30 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -346,7 +346,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.info()); + throw SubstituteGone(std::move(e.info())); } decompressor->finish(); @@ -370,7 +370,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); getFile(narInfoFile, - {[=](std::future<std::optional<std::string>> fut) { + {[=,this](std::future<std::optional<std::string>> fut) { try { auto data = fut.get(); @@ -502,22 +502,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe writeNarInfo(narInfo); } -std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) +std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath & path) { - auto drvPath = path; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return std::nullopt; - drvPath = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - - auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 8c82e2387..abd92a83c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -129,7 +129,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::optional<std::string> getBuildLog(const StorePath & path) override; + std::optional<std::string> getBuildLogExact(const StorePath & path) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..2021d0023 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -39,7 +39,6 @@ #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#include <sys/personality.h> #include <sys/mman.h> #include <sched.h> #include <sys/param.h> @@ -64,7 +63,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) @@ -83,7 +82,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) @@ -134,7 +133,7 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, ex); + done(BuildResult::TimedOut, {}, std::move(ex)); } @@ -143,18 +142,12 @@ void DerivationGoal::work() (this->*state)(); } -void DerivationGoal::addWantedOutputs(const StringSet & outputs) +void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); + auto newWanted = wantedOutputs.union_(outputs); + if (!newWanted.isSubsetOf(wantedOutputs)) needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; + wantedOutputs = newWanted; } @@ -391,7 +384,7 @@ void DerivationGoal::repairClosure() auto outputs = queryDerivationOutputMap(); StorePathSet outputClosure; for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; + if (!wantedOutputs.contains(i.first)) continue; worker.store.computeFSClosure(i.second, outputClosure); } @@ -423,7 +416,7 @@ void DerivationGoal::repairClosure() if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); + addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); } if (waitees.empty()) { @@ -545,7 +538,8 @@ void DerivationGoal::inputsRealised() However, the impure derivations feature still relies on this fragile way of doing things, because its builds do not have a representation in the store, which is a usability problem - in itself */ + in itself. When implementing this logic entirely with lookups + make sure that they're cached. */ if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { worker.store.computeFSClosure(*outPath, inputPaths); } @@ -571,10 +565,6 @@ void DerivationGoal::inputsRealised() /* What type of derivation are we building? */ derivationType = drv->type(); - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1; - /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ @@ -589,12 +579,11 @@ void DerivationGoal::started() auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + "building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); } @@ -948,14 +937,6 @@ void DerivationGoal::buildDone() cleanupPostOutputsRegisteredModeNonCheck(); - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old @@ -984,7 +965,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, e); + done(st, {}, std::move(e)); return; } } @@ -1004,10 +985,15 @@ void DerivationGoal::resolvedFinished() StorePathSet outputPaths; - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); + // `wantedOutputs` might merely indicate “all the outputs” + auto realWantedOutputs = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return resolvedDrv.outputNames(); + }, + [&](const OutputsSpec::Names & names) { + return static_cast<std::set<std::string>>(names); + }, + }, wantedOutputs.raw()); for (auto & wantedOutput : realWantedOutputs) { auto initialOutput = get(initialOutputs, wantedOutput); @@ -1016,22 +1002,34 @@ void DerivationGoal::resolvedFinished() throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", worker.store.printStorePath(drvPath), wantedOutput); - auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); - if (!realisation) - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", - worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + + auto realisation = [&]{ + auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + if (take1) return *take1; + + /* The above `get` should work. But sateful tracking of + outputs in resolvedResult, this can get out of sync with the + store, which is our actual source of truth. For now we just + check the store directly if it fails. */ + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }); + if (take2) return *take2; + + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + }(); + if (drv->type().isPure()) { - auto newRealisation = *realisation; + auto newRealisation = realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; newRealisation.signatures.clear(); if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } - outputPaths.insert(realisation->outPath); - builtOutputs.emplace(realisation->id, *realisation); + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(realisation.id, realisation); } runPostBuildHook( @@ -1323,7 +1321,14 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() if (!drv->type().isPure()) return { false, {} }; bool checkHash = buildMode == bmRepair; - auto wantedOutputsLeft = wantedOutputs; + auto wantedOutputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast<StringSet>(names); + }, + }, wantedOutputs.raw()); DrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { @@ -1332,7 +1337,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() // this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) continue; auto & info = *initialOutput; - info.wanted = wantOutput(i.first, wantedOutputs); + info.wanted = wantedOutputs.contains(i.first); if (info.wanted) wantedOutputsLeft.erase(i.first); if (i.second) { @@ -1370,7 +1375,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } - // If we requested all the outputs via the empty set, we are always fine. + // If we requested all the outputs, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. if (!wantedOutputsLeft.empty()) @@ -1435,7 +1440,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 2d8bfd592..707e38b4b 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,6 +2,7 @@ #include "parsed-derivations.hh" #include "lock.hh" +#include "outputs-spec.hh" #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" @@ -55,7 +56,7 @@ struct DerivationGoal : public Goal /* The specific outputs that we need to build. Empty means all of them. */ - StringSet wantedOutputs; + OutputsSpec wantedOutputs; /* Mapping from input derivations + output names to actual store paths. This is filled in by waiteeDone() as each dependency @@ -115,11 +116,6 @@ struct DerivationGoal : public Goal BuildMode buildMode; - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr<Activity> act; @@ -133,10 +129,10 @@ struct DerivationGoal : public Goal std::string machineName; DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); virtual ~DerivationGoal(); @@ -147,7 +143,7 @@ struct DerivationGoal : public Goal void work() override; /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); + void addWantedOutputs(const OutputsSpec & outputs); /* The states. */ void getDerivation(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index bea7363db..2925fe3ca 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -30,7 +30,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod if (ex) logError(i->ex->info()); else - ex = i->ex; + ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath); @@ -40,7 +40,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod if (failed.size() == 1 && ex) { ex->status = worker.exitStatus(); - throw *ex; + throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); @@ -80,7 +80,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { Worker worker(*this, *this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); try { worker.run(Goals{goal}); @@ -89,7 +89,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { .drvPath = drvPath }, + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, }; }; } @@ -109,7 +112,7 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { goal->ex->status = worker.exitStatus(); - throw *goal->ex; + throw std::move(*goal->ex); } else throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } @@ -130,7 +133,8 @@ void LocalStore::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); + // FIXME: Should just build the specific output we need. + goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); worker.run(goals); } else throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d2798888b..a961d8eed 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,6 +15,8 @@ #include "callback.hh" #include "json-utils.hh" #include "cgroup.hh" +#include "personality.hh" +#include "namespaces.hh" #include <regex> #include <queue> @@ -24,7 +26,6 @@ #include <termios.h> #include <unistd.h> #include <sys/mman.h> -#include <sys/utsname.h> #include <sys/resource.h> #include <sys/socket.h> @@ -37,7 +38,6 @@ #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> -#include <sys/personality.h> #include <sys/mman.h> #include <sched.h> #include <sys/param.h> @@ -168,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -206,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -230,7 +242,7 @@ void LocalDerivationGoal::tryLocalBuild() { outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, e); + done(BuildResult::InputRejected, {}, std::move(e)); return; } @@ -373,12 +385,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -int childEntry(void * arg) -{ - ((LocalDerivationGoal *) arg)->runChild(); - return 1; -} - #if __linux__ static void linkOrCopy(const Path & from, const Path & to) { @@ -664,7 +670,8 @@ void LocalDerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - chownToBuilder(chrootRootDir + "/etc"); + if (parsedDrv->useUidRange()) + chownToBuilder(chrootRootDir + "/etc"); if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); @@ -889,12 +896,7 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; + usingUserNamespace = userNamespacesSupported(); Pid helper = startProcess([&]() { @@ -909,76 +911,21 @@ void LocalDerivationGoal::startBuilder() if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) - flags |= CLONE_NEWNET; + options.cloneFlags |= CLONE_NEWNET; if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -1046,9 +993,6 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { runChild(); }); @@ -1460,7 +1404,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo unknown, downloadSize, narSize); } - virtual std::optional<std::string> getBuildLog(const StorePath & path) override + virtual std::optional<std::string> getBuildLogExact(const StorePath & path) override { return std::nullopt; } virtual void addBuildLog(const StorePath & path, std::string_view log) override @@ -1517,8 +1461,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + daemon::NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -1908,6 +1851,10 @@ void LocalDerivationGoal::runChild() } } + /* Make /etc unwritable */ + if (!parsedDrv->useUidRange()) + chmod_(chrootRootDir + "/etc", 0555); + /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount namespace. This means that the call to setns() in @@ -1964,33 +1911,7 @@ void LocalDerivationGoal::runChild() /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if ((drv->platform == "i686-linux" - && (settings.thisSystem == "x86_64-linux" - || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) - || drv->platform == "armv7l-linux" - || drv->platform == "armv6l-linux") - { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set 32-bit personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif + setPersonality(drv->platform); /* Disable core dumps by default. */ struct rlimit limit = { 0, RLIM_INFINITY }; @@ -2077,10 +1998,14 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + sandboxProfile += + #include "sandbox-defaults.sb" + ; if (!derivationType.isSandboxed()) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; + sandboxProfile += + #include "sandbox-network.sb" + ; /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; @@ -2123,7 +2048,9 @@ void LocalDerivationGoal::runChild() sandboxProfile += additionalSandboxProfile; } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + sandboxProfile += + #include "sandbox-minimal.sb" + ; debug("Generated sandbox profile:"); debug(sandboxProfile); @@ -2148,8 +2075,6 @@ void LocalDerivationGoal::runChild() args.push_back(sandboxFile); args.push_back("-D"); args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); if (allowLocalNetworking) { args.push_back("-D"); args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); @@ -2260,7 +2185,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -2347,11 +2271,28 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get<bool>(); + } + } + } - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, @@ -2688,10 +2629,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); } - if (curRound == nrRounds) { - localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() + worker.markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2720,61 +2659,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Apply output checks. */ checkOutputs(infos); - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - buildResult.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - printError(hint); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - renameFile(path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return {}; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -2816,7 +2700,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - if (wantOutput(outputName, wantedOutputs)) + if (wantedOutputs.contains(outputName)) builtOutputs.emplace(thisRealisation.id, thisRealisation); } diff --git a/src/libstore/build/personality.cc b/src/libstore/build/personality.cc new file mode 100644 index 000000000..4ad477869 --- /dev/null +++ b/src/libstore/build/personality.cc @@ -0,0 +1,44 @@ +#include "personality.hh" +#include "globals.hh" + +#if __linux__ +#include <sys/utsname.h> +#include <sys/personality.h> +#endif + +#include <cstring> + +namespace nix { + +void setPersonality(std::string_view system) +{ +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if ((system == "i686-linux" + && (std::string_view(SYSTEM) == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || system == "armv7l-linux" + || system == "armv6l-linux") + { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set 32-bit personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +#endif +} + +} diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh new file mode 100644 index 000000000..30e4f4062 --- /dev/null +++ b/src/libstore/build/personality.hh @@ -0,0 +1,11 @@ +#pragma once + +#include <string> + +namespace nix { + +void setPersonality(std::string_view system); + +} + + diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/build/sandbox-defaults.sb index d9d710559..77f013aea 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/build/sandbox-defaults.sb @@ -1,3 +1,5 @@ +R""( + (define TMPDIR (param "_GLOBAL_TMP_DIR")) (deny default) @@ -104,3 +106,5 @@ (subpath "/System/Library/Apple/usr/libexec/oah") (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) + +)"" diff --git a/src/libstore/sandbox-minimal.sb b/src/libstore/build/sandbox-minimal.sb index 65f5108b3..976a1f636 100644 --- a/src/libstore/sandbox-minimal.sb +++ b/src/libstore/build/sandbox-minimal.sb @@ -1,5 +1,9 @@ +R""( + (allow default) ; Disallow creating setuid/setgid binaries, since that ; would allow breaking build user isolation. (deny file-write-setugid) + +)"" diff --git a/src/libstore/sandbox-network.sb b/src/libstore/build/sandbox-network.sb index 19e9eea9a..335edbaed 100644 --- a/src/libstore/sandbox-network.sb +++ b/src/libstore/build/sandbox-network.sb @@ -1,3 +1,5 @@ +R""( + ; Allow local and remote network traffic. (allow network* (local ip) (remote ip)) @@ -18,3 +20,5 @@ ; Allow access to trustd. (allow mach-lookup (global-name "com.apple.trustd")) (allow mach-lookup (global-name "com.apple.trustd.agent")) + +)"" diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b192fbc77..f775f8486 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -42,7 +42,7 @@ Worker::~Worker() std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( const StorePath & drvPath, - const StringSet & wantedOutputs, + const OutputsSpec & wantedOutputs, std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) { std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath]; @@ -59,7 +59,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) + const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) @@ -70,7 +70,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) + const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) @@ -276,7 +276,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a1e036a96..6d68d3cf1 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -140,15 +140,15 @@ public: /* derivation goal */ private: std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, + const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); public: std::shared_ptr<DerivationGoal> makeDerivationGoal( const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 47458a388..b1fbda13d 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error( "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or type 'nix profile install --help' if using 'nix profile' to find out how" + "or type 'nix profile install --help' if using 'nix profile' to find out how " "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 48dd5c247..5e6fd011f 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -222,7 +222,8 @@ struct ClientSettings else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; @@ -235,10 +236,13 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString<Paths>(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name - || name == settings.buildRepeat.name || name == settings.maxSilentTime.name || name == settings.pollInterval.name || name == "connect-timeout" @@ -525,7 +529,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -542,7 +553,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); @@ -981,8 +994,7 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function<void(Store &)> authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr; @@ -1025,10 +1037,6 @@ void processConnection( try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..8c765615c 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -13,11 +13,6 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function<void(Store &)> authHook); + RecursiveFlag recursive); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe99c3c5e..05dc9a3cc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -5,6 +5,7 @@ #include "worker-protocol.hh" #include "fs-accessor.hh" #include <boost/container/small_vector.hpp> +#include <nlohmann/json.hpp> namespace nix { @@ -448,7 +449,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, // FIXME: remove -bool isDerivation(const std::string & fileName) +bool isDerivation(std::string_view fileName) { return hasSuffix(fileName, drvExtension); } @@ -688,12 +689,6 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & } -bool wantOutput(const std::string & output, const std::set<std::string> & wanted) -{ - return wanted.empty() || wanted.find(output) != wanted.end(); -} - - static DerivationOutput readDerivationOutput(Source & in, const Store & store) { const auto pathS = readString(in); @@ -896,4 +891,63 @@ std::optional<BasicDerivation> Derivation::tryResolve( const Hash impureOutputHash = hashString(htSHA256, "impure"); +nlohmann::json DerivationOutput::toJSON( + const Store & store, std::string_view drvName, std::string_view outputName) const +{ + nlohmann::json res = nlohmann::json::object(); + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doi) { + res["path"] = store.printStorePath(doi.path); + }, + [&](const DerivationOutput::CAFixed & dof) { + res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + res["hashAlgo"] = dof.hash.printMethodAlgo(); + res["hash"] = dof.hash.hash.to_string(Base16, false); + }, + [&](const DerivationOutput::CAFloating & dof) { + res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); + }, + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); + res["impure"] = true; + }, + }, raw()); + return res; +} + +nlohmann::json Derivation::toJSON(const Store & store) const +{ + nlohmann::json res = nlohmann::json::object(); + + { + nlohmann::json & outputsObj = res["outputs"]; + outputsObj = nlohmann::json::object(); + for (auto & [outputName, output] : outputs) { + outputsObj[outputName] = output.toJSON(store, name, outputName); + } + } + + { + auto& inputsList = res["inputSrcs"]; + inputsList = nlohmann::json ::array(); + for (auto & input : inputSrcs) + inputsList.emplace_back(store.printStorePath(input)); + } + + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json ::object(); + for (auto & input : inputDrvs) + inputDrvsObj[store.printStorePath(input.first)] = input.second; + } + + res["system"] = platform; + res["builder"] = builder; + res["args"] = args; + res["env"] = env; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index af198a767..8456b29e7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -13,6 +13,7 @@ namespace nix { +class Store; /* Abstract syntax of derivations. */ @@ -82,6 +83,11 @@ struct DerivationOutput : _DerivationOutputRaw inline const Raw & raw() const { return static_cast<const Raw &>(*this); } + + nlohmann::json toJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName) const; }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; @@ -209,6 +215,8 @@ struct Derivation : BasicDerivation Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + + nlohmann::json toJSON(const Store & store) const; }; @@ -224,7 +232,7 @@ StorePath writeDerivation(Store & store, Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove -bool isDerivation(const std::string & fileName); +bool isDerivation(std::string_view fileName); /* Calculate the name that will be used for the store path for this output. @@ -294,8 +302,6 @@ typedef std::map<StorePath, DrvHash> DrvHashes; // FIXME: global, though at least thread-safe. extern Sync<DrvHashes> drvHashes; -bool wantOutput(const std::string & output, const std::set<std::string> & wanted); - struct Source; struct Sink; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 88b59f615..e0d86a42f 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -19,12 +19,13 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const { res["drvPath"] = store->printStorePath(drvPath); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); - for (const auto& output : outputs) { - auto knownOutput = get(knownOutputs, output); - res["outputs"][output] = (knownOutput && *knownOutput) - ? store->printStorePath(**knownOutput) - : nullptr; + const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + for (const auto & [output, outputPathOpt] : outputMap) { + if (!outputs.contains(output)) continue; + if (outputPathOpt) + res["outputs"][output] = store->printStorePath(*outputPathOpt); + else + res["outputs"][output] = nullptr; } return res; } @@ -62,7 +63,7 @@ std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" - + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); + + outputs.to_string(); } std::string DerivedPath::to_string(const Store & store) const @@ -78,16 +79,12 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) { - size_t n = s.find("!"); - assert(n != s.npos); - auto drvPath = store.parseStorePath(s.substr(0, n)); - auto outputsS = s.substr(n + 1); - std::set<std::string> outputs; - if (outputsS != "*") - outputs = tokenizeString<std::set<std::string>>(outputsS, ","); - return {drvPath, outputs}; + return { + .drvPath = store.parseStorePath(drvS), + .outputs = OutputsSpec::parse(outputsS), + }; } DerivedPath DerivedPath::parse(const Store & store, std::string_view s) @@ -95,7 +92,7 @@ DerivedPath DerivedPath::parse(const Store & store, std::string_view s) size_t n = s.find("!"); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s); + : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 878696136..9e0cce377 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,8 +3,10 @@ #include "util.hh" #include "path.hh" #include "realisation.hh" +#include "outputs-spec.hh" +#include "comparator.hh" -#include <optional> +#include <variant> #include <nlohmann/json_fwd.hpp> @@ -26,8 +28,7 @@ struct DerivedPathOpaque { std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + GENERATE_CMP(DerivedPathOpaque, me->path); }; /** @@ -44,14 +45,13 @@ struct DerivedPathOpaque { */ struct DerivedPathBuilt { StorePath drvPath; - std::set<std::string> outputs; + OutputsSpec outputs; std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view); + static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref<Store> store) const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); }; using _DerivedPathRaw = std::variant< @@ -95,6 +95,8 @@ struct BuiltPathBuilt { nlohmann::json toJSON(ref<Store> store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); }; using _BuiltPathRaw = std::variant< diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5746c32a3..b25089ec3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -33,14 +33,6 @@ FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); -std::string resolveUri(std::string_view uri) -{ - if (uri.compare(0, 8, "channel:") == 0) - return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(uri); -} - struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; @@ -109,6 +101,7 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) @@ -142,9 +135,9 @@ struct curlFileTransfer : public FileTransfer } template<class T> - void fail(const T & e) + void fail(T && e) { - failEx(std::make_exception_ptr(e)); + failEx(std::make_exception_ptr(std::move(e))); } LambdaSink finalSink; @@ -472,7 +465,7 @@ struct curlFileTransfer : public FileTransfer fileTransfer.enqueueItem(shared_from_this()); } else - fail(exc); + fail(std::move(exc)); } } }; @@ -873,14 +866,4 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<st err.msg = hf; } -bool isUri(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - - } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 40e7cf52c..07d58f53a 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -125,9 +125,4 @@ public: FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args); }; -bool isUri(std::string_view s); - -/* Resolve deprecated 'channel:<foo>' URLs. */ -std::string resolveUri(std::string_view uri); - } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5d91829f1..996f26a95 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -77,60 +77,73 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot } -void LocalStore::addTempRoot(const StorePath & path) +void LocalStore::createTempRootsFile() { - auto state(_state.lock()); + auto fdTempRoots(_fdTempRoots.lock()); /* Create the temporary roots file for this process. */ - if (!state->fdTempRoots) { - - while (1) { - if (pathExists(fnTempRoots)) - /* It *must* be stale, since there can be no two - processes with the same pid. */ - unlink(fnTempRoots.c_str()); + if (*fdTempRoots) return; - state->fdTempRoots = openLockFile(fnTempRoots, true); + while (1) { + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); - debug("acquiring write lock on '%s'", fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + *fdTempRoots = openLockFile(fnTempRoots, true); - /* Check whether the garbage collector didn't get in our - way. */ - struct stat st; - if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError("statting '%1%'", fnTempRoots); - if (st.st_size == 0) break; + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(fdTempRoots->get(), ltWrite, true); - /* The garbage collector deleted this file before we could - get a lock. (It won't delete the file after we get a - lock.) Try again. */ - } + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(fdTempRoots->get(), &st) == -1) + throw SysError("statting '%1%'", fnTempRoots); + if (st.st_size == 0) break; + /* The garbage collector deleted this file before we could get + a lock. (It won't delete the file after we get a lock.) + Try again. */ } +} + - if (!state->fdGCLock) - state->fdGCLock = openGCLock(); +void LocalStore::addTempRoot(const StorePath & path) +{ + createTempRootsFile(); + + /* Open/create the global GC lock file. */ + { + auto fdGCLock(_fdGCLock.lock()); + if (!*fdGCLock) + *fdGCLock = openGCLock(); + } restart: - FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + /* Try to acquire a shared global GC lock (non-blocking). This + only succeeds if the garbage collector is not currently + running. */ + FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, ""); if (!gcLock.acquired) { /* We couldn't get a shared global GC lock, so the garbage collector is running. So we have to connect to the garbage collector and inform it about our root. */ - if (!state->fdRootsSocket) { + auto fdRootsSocket(_fdRootsSocket.lock()); + + if (!*fdRootsSocket) { auto socketPath = stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); - state->fdRootsSocket = createUnixDomainSocket(); + *fdRootsSocket = createUnixDomainSocket(); try { - nix::connect(state->fdRootsSocket.get(), socketPath); + nix::connect(fdRootsSocket->get(), socketPath); } catch (SysError & e) { /* The garbage collector may have exited, so we need to restart. */ if (e.errNo == ECONNREFUSED) { debug("GC socket connection refused"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } throw; @@ -139,9 +152,9 @@ void LocalStore::addTempRoot(const StorePath & path) try { debug("sending GC root '%s'", printStorePath(path)); - writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false); char c; - readFull(state->fdRootsSocket.get(), &c, 1); + readFull(fdRootsSocket->get(), &c, 1); assert(c == '1'); debug("got ack for GC root '%s'", printStorePath(path)); } catch (SysError & e) { @@ -149,20 +162,21 @@ void LocalStore::addTempRoot(const StorePath & path) restart. */ if (e.errNo == EPIPE || e.errNo == ECONNRESET) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } throw; } catch (EndOfFile & e) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } } - /* Append the store path to the temporary roots file. */ + /* Record the store path in the temporary roots file so it will be + seen by a future run of the garbage collector. */ auto s = printStorePath(path) + '\0'; - writeFull(state->fdTempRoots.get(), s); + writeFull(_fdTempRoots.lock()->get(), s); } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b7f55cae7..8e33a3dec 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -222,19 +222,19 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s .longName = name, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } @@ -291,4 +291,18 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static bool initLibStoreDone = false; + +void assertLibStoreInitialized() { + if (!initLibStoreDone) { + printError("The program must call nix::initNix() before calling any libstore library functions."); + abort(); + }; +} + +void initLibStore() { + initLibStoreDone = true; +} + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca72ad31e..0a4912f67 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -279,9 +279,12 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. - )"}; + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. + + Defaults to `nixbld` when running as root, *empty* otherwise. + )", + {}, false}; Setting<bool> autoAllocateUids{this, false, "auto-allocate-uids", R"( @@ -326,7 +329,7 @@ public: Whether to execute builds inside cgroups. This is only supported on Linux. - Cgroups are required and enabled automatically for derivations + Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. > **Warning** @@ -373,11 +376,6 @@ public: )", {"build-max-log-size"}}; - /* When buildRepeat > 0 and verboseBuild == true, whether to print - repeated builds (i.e. builds other than the first one) to - stderr. Hack to prevent Hydra logs from being polluted. */ - bool printRepeatedBuilds = true; - Setting<unsigned int> pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; @@ -493,6 +491,9 @@ public: for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will only be mounted in the sandbox if it exists in the host filesystem. + If the source is in the Nix store, then its closure will be added to + the sandbox as well. + Depending on how Nix was built, the default value for this option may be empty or provide `/bin/sh` as a bind-mount of `bash`. )", @@ -501,19 +502,6 @@ public: Setting<bool> sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting<size_t> buildRepeat{ - this, 0, "repeat", - R"( - How many times to repeat builds to check whether they are - deterministic. The default value is 0. If the value is non-zero, - every build is repeated the specified number of times. If the - contents of any of the runs differs from the previous ones and - `enforce-determinism` is true, the build is rejected and the - resulting store paths are not registered as “valid” in Nix’s - database. - )", - {"build-repeat"}}; - #if __linux__ Setting<std::string> sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", @@ -577,20 +565,20 @@ public: configuration file, and cannot be passed at the command line. )"}; - Setting<bool> enforceDeterminism{ - this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output. See `repeat`."}; - Setting<Strings> trustedPublicKeys{ this, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a substituter): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -686,13 +674,14 @@ public: independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. - Nix will copy a store path from a remote store only if one - of the following is true: + At least one of the following conditions must be met for Nix to use + a substituter: - - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) + - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described + in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; @@ -707,24 +696,6 @@ public: )", {"trusted-binary-caches"}}; - Setting<Strings> trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting<unsigned int> ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -747,18 +718,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting<Strings> allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting<bool> printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; @@ -986,6 +945,27 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; + + Setting<bool> useXDGBaseDirectories{ + this, false, "use-xdg-base-directories", + R"( + If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`. + The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md). + + [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + + > **Warning** + > This changes the location of some well-known symlinks that Nix creates, which might break tools that rely on the old, non-XDG-conformant locations. + + In particular, the following locations change: + + | Old | New | + |-------------------|--------------------------------| + | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | + | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | + | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + )" + }; }; @@ -1003,4 +983,12 @@ std::vector<Path> getUserConfigFiles(); extern const std::string nixVersion; +/* NB: This is not sufficient. You need to call initNix() */ +void initLibStore(); + +/* It's important to initialize before doing _anything_, which is why we + call upon the programmer to handle this correctly. However, we only add + this in a key locations, so as not to litter the code. */ +void assertLibStoreInitialized(); + } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 73bcd6e81..1479822a9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -56,7 +56,7 @@ public: void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dd34b19c6..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -134,7 +134,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); @@ -255,8 +254,8 @@ private: << settings.maxLogSize; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) conn.to - << settings.buildRepeat - << settings.enforceDeterminism; + << 0 // buildRepeat hasn't worked for ages anyway + << 0; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { conn.to << ((int) settings.keepFailed); @@ -279,7 +278,12 @@ public: conn->to.flush(); - BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; + BuildResult status { + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c5ae7536f..b224fc3e9 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -87,20 +87,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) const std::string LocalFSStore::drvsLogDir = "drvs"; -std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_) +std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path) { - auto path = path_; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - if (!info->deriver) return std::nullopt; - path = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - auto baseName = path.to_string(); for (int j = 0; j < 2; j++) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index e6fb3201a..947707341 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -50,7 +50,7 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::optional<std::string> getBuildLog(const StorePath & path) override; + std::optional<std::string> getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b67668e52..82edaa9bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -91,6 +91,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) if (!lockFile(lockFd.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(lockFd.get(), ltWrite, true); } @@ -200,8 +201,6 @@ LocalStore::LocalStore(const Params & params) throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -299,6 +298,7 @@ LocalStore::LocalStore(const Params & params) if (!lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); + lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); } @@ -439,9 +439,9 @@ LocalStore::~LocalStore() } try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = -1; + auto fdTempRoots(_fdTempRoots.lock()); + if (*fdTempRoots) { + *fdTempRoots = -1; unlink(fnTempRoots.c_str()); } } catch (...) { @@ -1822,20 +1822,6 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4579c2f62..a84eb7c26 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -59,15 +59,6 @@ private: struct Stmts; std::unique_ptr<Stmts> stmts; - /* The global GC lock */ - AutoCloseFD fdGCLock; - - /* The file to which we write our temporary roots. */ - AutoCloseFD fdTempRoots; - - /* Connection to the garbage collector. */ - AutoCloseFD fdRootsSocket; - /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; @@ -156,6 +147,21 @@ public: void addTempRoot(const StorePath & path) override; +private: + + void createTempRootsFile(); + + /* The file to which we write our temporary roots. */ + Sync<AutoCloseFD> _fdTempRoots; + + /* The global GC lock. */ + Sync<AutoCloseFD> _fdGCLock; + + /* Connection to the garbage collector. */ + Sync<AutoCloseFD> _fdRootsSocket; + +public: + void addIndirectRoot(const Path & path) override; private: @@ -275,8 +281,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 8f28bec6c..e5e24501e 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -13,10 +13,6 @@ ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif -ifdef HOST_DARWIN -libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb -endif - $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 2858137d6..4fe1fcf56 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -123,8 +123,12 @@ struct AutoUserLock : UserLock std::vector<gid_t> getSupplementaryGIDs() override { return {}; } - static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useChroot) + static std::unique_ptr<UserLock> acquire(uid_t nrIds, bool useUserNamespace) { + #if !defined(__linux__) + useUserNamespace = false; + #endif + settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); assert(settings.uidCount % maxIdsPerBuild == 0); @@ -157,7 +161,7 @@ struct AutoUserLock : UserLock auto lock = std::make_unique<AutoUserLock>(); lock->fdUserLock = std::move(fd); lock->firstUid = firstUid; - if (useChroot) + if (useUserNamespace) lock->firstGid = firstUid; else { struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -174,10 +178,10 @@ struct AutoUserLock : UserLock } }; -std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot) +std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace) { if (settings.autoAllocateUids) - return AutoUserLock::acquire(nrIds, useChroot); + return AutoUserLock::acquire(nrIds, useUserNamespace); else return SimpleUserLock::acquire(); } @@ -185,7 +189,7 @@ std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot) bool useBuildUsers() { #if __linux__ - static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; + static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0; return b; #elif __APPLE__ static bool b = settings.buildUsersGroup != "" && getuid() == 0; diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 49ad86de7..7f1934510 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -31,7 +31,7 @@ struct UserLock /* Acquire a user lock for a UID range of size `nrIds`. Note that this may return nullptr if no user is available. */ -std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useChroot); +std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc new file mode 100644 index 000000000..8a26832ab --- /dev/null +++ b/src/libstore/log-store.cc @@ -0,0 +1,12 @@ +#include "log-store.hh" + +namespace nix { + +std::optional<std::string> LogStore::getBuildLog(const StorePath & path) { + auto maybePath = getBuildDerivationPath(path); + if (!maybePath) + return std::nullopt; + return getBuildLogExact(maybePath.value()); +} + +} diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index ff1b92e17..e4d95bab6 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -11,7 +11,9 @@ struct LogStore : public virtual Store /* Return the build log of the specified store path, if available, or null otherwise. */ - virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0; + std::optional<std::string> getBuildLog(const StorePath & path); + + virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index fb985c97b..b28768459 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -185,7 +185,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, knownOutputPaths = false; break; } - if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) + if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; @@ -301,4 +301,47 @@ std::map<DrvOutput, StorePath> drvOutputReferences( return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + OutputPathMap outputs; + auto drv = evalStore.readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(store, drv); + auto drvOutputs = drv.outputsAndOptPaths(store); + auto outputNames = std::visit(overloaded { + [&](const OutputsSpec::All &) { + StringSet names; + for (auto & [outputName, _] : drv.outputs) + names.insert(outputName); + return names; + }, + [&](const OutputsSpec::Names & names) { + return static_cast<std::set<std::string>>(names); + }, + }, bfd.outputs.raw()); + for (auto & output : outputNames) { + auto outputHash = get(outputHashes, output); + if (!outputHash) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + DrvOutput outputId { *outputHash, output }; + auto realisation = store.queryRealisation(outputId); + if (!realisation) + throw MissingRealisation(outputId); + outputs.insert_or_assign(output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + auto drvOutput = get(drvOutputs, output); + assert(drvOutput); + assert(drvOutput->second); + outputs.insert_or_assign(output, *drvOutput->second); + } + } + return outputs; +} + } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index f4ea739b0..2645f468b 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -84,11 +84,10 @@ public: Sync<State> _state; - NarInfoDiskCacheImpl() + NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite") { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -98,7 +97,7 @@ public: state->db.exec(schema); state->insertCache.create(state->db, - "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); state->queryCache.create(state->db, "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); @@ -166,38 +165,71 @@ public: return i->second; } - void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override +private: + + std::optional<Cache> queryCacheRaw(State & state, const std::string & uri) { - retrySQLite<void>([&]() { + auto i = state.caches.find(uri); + if (i == state.caches.end()) { + auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl)); + if (!queryCache.next()) + return std::nullopt; + auto cache = Cache { + .id = (int) queryCache.getInt(0), + .storeDir = queryCache.getStr(1), + .wantMassQuery = queryCache.getInt(2) != 0, + .priority = (int) queryCache.getInt(3), + }; + state.caches.emplace(uri, cache); + } + return getCache(state, uri); + } + +public: + int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override + { + return retrySQLite<int>([&]() { auto state(_state.lock()); + SQLiteTxn txn(state->db); + + // To avoid the race, we have to check if maybe someone hasn't yet created + // the cache for this URI in the meantime. + auto cache(queryCacheRaw(*state, uri)); + + if (cache) + return cache->id; + + Cache ret { + .id = -1, // set below + .storeDir = storeDir, + .wantMassQuery = wantMassQuery, + .priority = priority, + }; + + { + auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); + assert(r.next()); + ret.id = (int) r.getInt(0); + } - // FIXME: race + state->caches[uri] = ret; - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + txn.commit(); + return ret.id; }); } - std::optional<CacheInfo> cacheExists(const std::string & uri) override + std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override { return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> { auto state(_state.lock()); - - auto i = state->caches.find(uri); - if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl)); - if (!queryCache.next()) - return std::nullopt; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); - } - - auto & cache(getCache(*state, uri)); - + auto cache(queryCacheRaw(*state, uri)); + if (!cache) + return std::nullopt; return CacheInfo { - .wantMassQuery = cache.wantMassQuery, - .priority = cache.priority + .id = cache->id, + .wantMassQuery = cache->wantMassQuery, + .priority = cache->priority }; }); } @@ -359,4 +391,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache() return cache; } +ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath) +{ + return make_ref<NarInfoDiskCacheImpl>(dbPath); +} + } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 2dcaa76a4..4877f56d8 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -13,16 +13,17 @@ public: virtual ~NarInfoDiskCache() { } - virtual void createCache(const std::string & uri, const Path & storeDir, + virtual int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; struct CacheInfo { + int id; bool wantMassQuery; int priority; }; - virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0; + virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0; virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; @@ -45,4 +46,6 @@ public: multiple threads. */ ref<NarInfoDiskCache> getNarInfoDiskCache(); +ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath); + } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 6d67b1e03..dc42d0bca 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc new file mode 100644 index 000000000..e26c38138 --- /dev/null +++ b/src/libstore/outputs-spec.cc @@ -0,0 +1,195 @@ +#include <regex> +#include <nlohmann/json.hpp> + +#include "util.hh" +#include "regex-combinators.hh" +#include "outputs-spec.hh" +#include "path-regex.hh" + +namespace nix { + +bool OutputsSpec::contains(const std::string & outputName) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & outputNames) { + return outputNames.count(outputName) > 0; + }, + }, raw()); +} + +static std::string outputSpecRegexStr = + regex::either( + regex::group(R"(\*)"), + regex::group(regex::list(nameRegexStr))); + +std::optional<OutputsSpec> OutputsSpec::parseOpt(std::string_view s) +{ + static std::regex regex(std::string { outputSpecRegexStr }); + + std::smatch match; + std::string s2 { s }; // until some improves std::regex + if (!std::regex_match(s2, match, regex)) + return std::nullopt; + + if (match[1].matched) + return { OutputsSpec::All {} }; + + if (match[2].matched) + return OutputsSpec::Names { tokenizeString<StringSet>(match[2].str(), ",") }; + + assert(false); +} + + +OutputsSpec OutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("invalid outputs specifier '%s'", s); + return *spec; +} + + +std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsSpec::parseOpt(std::string_view s) +{ + auto found = s.rfind('^'); + + if (found == std::string::npos) + return std::pair { s, ExtendedOutputsSpec::Default {} }; + + auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); + if (!specOpt) + return std::nullopt; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; +} + + +std::pair<std::string_view, ExtendedOutputsSpec> ExtendedOutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("invalid extended outputs specifier '%s'", s); + return *spec; +} + + +std::string OutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> std::string { + return "*"; + }, + [&](const OutputsSpec::Names & outputNames) -> std::string { + return concatStringsSep(",", outputNames); + }, + }, raw()); +} + + +std::string ExtendedOutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) -> std::string { + return ""; + }, + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string { + return "^" + outputSpec.to_string(); + }, + }, raw()); +} + + +OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All { }; + }, + [&](const OutputsSpec::Names & theseNames) -> OutputsSpec { + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All {}; + }, + [&](const OutputsSpec::Names & thoseNames) -> OutputsSpec { + OutputsSpec::Names ret = theseNames; + ret.insert(thoseNames.begin(), thoseNames.end()); + return ret; + }, + }, that.raw()); + }, + }, raw()); +} + + +bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & thoseNames) { + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return false; + }, + [&](const OutputsSpec::Names & theseNames) { + bool ret = true; + for (auto & o : theseNames) + if (thoseNames.count(o) == 0) + ret = false; + return ret; + }, + }, raw()); + }, + }, that.raw()); +} + +} + +namespace nlohmann { + +using namespace nix; + +OutputsSpec adl_serializer<OutputsSpec>::from_json(const json & json) { + auto names = json.get<StringSet>(); + if (names == StringSet({"*"})) + return OutputsSpec::All {}; + else + return OutputsSpec::Names { std::move(names) }; +} + +void adl_serializer<OutputsSpec>::to_json(json & json, OutputsSpec t) { + std::visit(overloaded { + [&](const OutputsSpec::All &) { + json = std::vector<std::string>({"*"}); + }, + [&](const OutputsSpec::Names & names) { + json = names; + }, + }, t.raw()); +} + + +ExtendedOutputsSpec adl_serializer<ExtendedOutputsSpec>::from_json(const json & json) { + if (json.is_null()) + return ExtendedOutputsSpec::Default {}; + else { + return ExtendedOutputsSpec::Explicit { json.get<OutputsSpec>() }; + } +} + +void adl_serializer<ExtendedOutputsSpec>::to_json(json & json, ExtendedOutputsSpec t) { + std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) { + json = nullptr; + }, + [&](const ExtendedOutputsSpec::Explicit & e) { + adl_serializer<OutputsSpec>::to_json(json, e); + }, + }, t.raw()); +} + +} diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh new file mode 100644 index 000000000..46bc35ebc --- /dev/null +++ b/src/libstore/outputs-spec.hh @@ -0,0 +1,95 @@ +#pragma once + +#include <cassert> +#include <optional> +#include <set> +#include <variant> + +#include "json-impls.hh" + +namespace nix { + +struct OutputNames : std::set<std::string> { + using std::set<std::string>::set; + + /* These need to be "inherited manually" */ + + OutputNames(const std::set<std::string> & s) + : std::set<std::string>(s) + { assert(!empty()); } + + OutputNames(std::set<std::string> && s) + : std::set<std::string>(s) + { assert(!empty()); } + + /* This set should always be non-empty, so we delete this + constructor in order make creating empty ones by mistake harder. + */ + OutputNames() = delete; +}; + +struct AllOutputs : std::monostate { }; + +typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw; + +struct OutputsSpec : _OutputsSpecRaw { + using Raw = _OutputsSpecRaw; + using Raw::Raw; + + /* Force choosing a variant */ + OutputsSpec() = delete; + + using Names = OutputNames; + using All = AllOutputs; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + inline Raw & raw() { + return static_cast<Raw &>(*this); + } + + bool contains(const std::string & output) const; + + /* Create a new OutputsSpec which is the union of this and that. */ + OutputsSpec union_(const OutputsSpec & that) const; + + /* Whether this OutputsSpec is a subset of that. */ + bool isSubsetOf(const OutputsSpec & outputs) const; + + /* Parse a string of the form 'output1,...outputN' or + '*', returning the outputs spec. */ + static OutputsSpec parse(std::string_view s); + static std::optional<OutputsSpec> parseOpt(std::string_view s); + + std::string to_string() const; +}; + +struct DefaultOutputs : std::monostate { }; + +typedef std::variant<DefaultOutputs, OutputsSpec> _ExtendedOutputsSpecRaw; + +struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { + using Raw = _ExtendedOutputsSpecRaw; + using Raw::Raw; + + using Default = DefaultOutputs; + using Explicit = OutputsSpec; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + /* Parse a string of the form 'prefix^output1,...outputN' or + 'prefix^*', returning the prefix and the extended outputs spec. */ + static std::pair<std::string_view, ExtendedOutputsSpec> parse(std::string_view s); + static std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> parseOpt(std::string_view s); + + std::string to_string() const; +}; + +} + +JSON_IMPL(OutputsSpec) +JSON_IMPL(ExtendedOutputsSpec) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fda55b2b6..bd55a9d06 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,80 @@ namespace nix { +std::string ValidPathInfo::fingerprint(const Store & store) const +{ + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", + store.printStorePath(path)); + return + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(Base32, true) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", store.printStorePathSet(references)); +} + + +void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint(store))); +} + + +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + if (! ca) return false; + + auto caPath = std::visit(overloaded { + [&](const TextHash & th) { + return store.makeTextPath(path.name(), th.hash, references); + }, + [&](const FixedOutputHash & fsh) { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); + } + }, *ca); + + bool res = caPath == path; + + if (!res) + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + + return res; +} + + +size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const +{ + if (isContentAddressed(store)) return maxSigs; + + size_t good = 0; + for (auto & sig : sigs) + if (checkSignature(store, publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(store), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(std::string(r.to_string())); + return refs; +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -24,6 +98,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned return info; } + void ValidPathInfo::write( Sink & sink, const Store & store, diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b4b54e593..a7fcbd232 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -81,12 +81,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits<size_t>::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh new file mode 100644 index 000000000..6893c3876 --- /dev/null +++ b/src/libstore/path-regex.hh @@ -0,0 +1,7 @@ +#pragma once + +namespace nix { + +static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; + +} diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index d6d67ea05..869b490ad 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,6 +1,5 @@ #include "path-with-outputs.hh" #include "store-api.hh" -#include "nlohmann/json.hpp" #include <regex> @@ -16,10 +15,14 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { - if (!outputs.empty() || path.isDerivation()) - return DerivedPath::Built { path, outputs }; - else + if (!outputs.empty()) { + return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + } else if (path.isDerivation()) { + assert(outputs.empty()); + return DerivedPath::Built { path, OutputsSpec::All { } }; + } else { return DerivedPath::Opaque { path }; + } } @@ -42,7 +45,18 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv return StorePathWithOutputs { bo.path }; }, [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> { - return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + return StorePathWithOutputs { + .path = bfd.drvPath, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast<StringSet>(outputs); + }, + }, bfd.outputs.raw()), + }; }, }, p.raw()); } @@ -53,8 +67,8 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s) size_t n = s.find("!"); return n == s.npos ? std::make_pair(s, std::set<std::string>()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ",")); + : std::make_pair(s.substr(0, n), + tokenizeString<std::set<std::string>>(s.substr(n + 1), ",")); } @@ -71,57 +85,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } -std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s) -{ - static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); - - std::smatch match; - if (!std::regex_match(s, match, regex)) - return {s, DefaultOutputs()}; - - if (match[3].matched) - return {match[1], AllOutputs()}; - - return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")}; -} - -std::string printOutputsSpec(const OutputsSpec & outputsSpec) -{ - if (std::get_if<DefaultOutputs>(&outputsSpec)) - return ""; - - if (std::get_if<AllOutputs>(&outputsSpec)) - return "^*"; - - if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - return "^" + concatStringsSep(",", *outputNames); - - assert(false); -} - -void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec) -{ - if (std::get_if<DefaultOutputs>(&outputsSpec)) - json = nullptr; - - else if (std::get_if<AllOutputs>(&outputsSpec)) - json = std::vector<std::string>({"*"}); - - else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - json = *outputNames; -} - -void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec) -{ - if (json.is_null()) - outputsSpec = DefaultOutputs(); - else { - auto names = json.get<OutputNames>(); - if (names == OutputNames({"*"})) - outputsSpec = AllOutputs(); - else - outputsSpec = names; - } -} - } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0cb5eb223..5d25656a5 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,13 +1,17 @@ #pragma once -#include <variant> - #include "path.hh" #include "derived-path.hh" -#include "nlohmann/json_fwd.hpp" namespace nix { +/* This is a deprecated old type just for use by the old CLI, and older + versions of the RPC protocols. In new code don't use it; you want + `DerivedPath` instead. + + `DerivedPath` is better because it handles more cases, and does so more + explicitly without devious punning tricks. +*/ struct StorePathWithOutputs { StorePath path; @@ -33,25 +37,4 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); -typedef std::set<std::string> OutputNames; - -struct AllOutputs { - bool operator < (const AllOutputs & _) const { return false; } -}; - -struct DefaultOutputs { - bool operator < (const DefaultOutputs & _) const { return false; } -}; - -typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec; - -/* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the outputs spec. */ -std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s); - -std::string printOutputsSpec(const OutputsSpec & outputsSpec); - -void to_json(nlohmann::json &, const OutputsSpec &); -void from_json(const nlohmann::json &, OutputsSpec &); - } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 392db225e..46be54281 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -8,8 +8,10 @@ static void checkName(std::string_view path, std::string_view name) { if (name.empty()) throw BadStorePath("store path '%s' has an empty name", path); - if (name.size() > 211) - throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + if (name.size() > StorePath::MaxPathLen) + throw BadStorePath("store path '%s' has a name longer than '%d characters", + StorePath::MaxPathLen, path); + // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 77fd0f8dc..1e5579b90 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,11 +1,11 @@ #pragma once -#include "content-address.hh" +#include <string_view> + #include "types.hh" namespace nix { -class Store; struct Hash; class StorePath @@ -17,6 +17,8 @@ public: /* Size of the hash part of store paths, in base-32 characters. */ constexpr static size_t HashLen = 32; // i.e. 160 bits + constexpr static size_t MaxPathLen = 211; + StorePath() = delete; StorePath(std::string_view baseName); @@ -64,9 +66,6 @@ public: typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -typedef std::map<std::string, StorePath> OutputPathMap; - -typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..c551c5f3e 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -280,16 +280,24 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = createNixStateDir() + "/profiles"; + createDirs(profileRoot); + return profileRoot; +} + + Path getDefaultProfile() { - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { + auto profile = + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : profilesDir() + "/profile"; if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + replaceSymlink(profile, profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..fbf95b850 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,8 +68,13 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); -/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create - it. */ +/* Creates and returns the path to a directory suitable for storing the user’s + profiles. */ +Path profilesDir(); + +/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ + nix/profile if XDG Base Directory Support is enabled), and create if doesn't + exist */ Path getDefaultProfile(); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 9070a6ee2..48d0283de 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,5 +1,8 @@ #pragma once +#include <variant> + +#include "hash.hh" #include "path.hh" #include <nlohmann/json_fwd.hpp> #include "comparator.hh" @@ -7,6 +10,8 @@ namespace nix { +class Store; + struct DrvOutput { // The hash modulo of the derivation Hash drvHash; @@ -93,4 +98,14 @@ struct RealisedPath { GENERATE_CMP(RealisedPath, me->raw); }; +class MissingRealisation : public Error +{ +public: + MissingRealisation(DrvOutput & outputId) + : Error( "cannot operate on an output of the " + "unbuilt derivation '%s'", + outputId.to_string()) + {} +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 96a29155c..d1296627a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -447,7 +448,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.info()); + throw InvalidPath(std::move(e.info())); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { @@ -867,8 +868,8 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drv = evalStore->readDerivation(bfd.drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - const auto drvOutputs = drv.outputsAndOptPaths(*this); - for (auto & output : bfd.outputs) { + auto built = resolveDerivedPath(*this, bfd, &*evalStore); + for (auto & [output, outputPath] : built) { auto outputHash = get(outputHashes, output); if (!outputHash) throw Error( @@ -879,22 +880,14 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults( auto realisation = queryRealisation(outputId); if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); + throw MissingRealisation(outputId); res.builtOutputs.emplace(realisation->id, *realisation); } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - const auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); res.builtOutputs.emplace( outputId, Realisation { .id = outputId, - .outPath = *drvOutput->second, + .outPath = outputPath, }); } } @@ -918,7 +911,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; + BuildResult res { + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 844553ad3..8d76eee99 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { - if (auto cacheInfo = diskCache->cacheExists(getUri())) { + if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 2090beabd..871f2f3be 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -8,12 +8,15 @@ namespace nix { -SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf) - : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) +SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf) + : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset) { - err.msg = hintfmt("%s: %s (in '%s')", + auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": "; + err.msg = hintfmt("%s: %s%s, %s (in '%s')", normaltxt(hf.str()), + offsetStr, sqlite3_errstr(extendedErrNo), + errMsg, path ? path : "(in-memory)"); } @@ -21,11 +24,13 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); + int offset = sqlite3_error_offset(db); auto path = sqlite3_db_filename(db, nullptr); + auto errMsg = sqlite3_errmsg(db); if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - auto exp = SQLiteBusy(path, err, exterr, std::move(hf)); + auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf)); exp.err.msg = hintfmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" @@ -33,22 +38,40 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor path ? path : "(in-memory)"); throw exp; } else - throw SQLiteError(path, err, exterr, std::move(hf)); + throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } +static void traceSQL(void * x, const char * sql) +{ + // wacky delimiters: + // so that we're quite unambiguous without escaping anything + // notice instead of trace: + // so that this can be enabled without getting the firehose in our face. + notice("SQL<[%1%]>", sql); +}; + SQLite::SQLite(const Path & path, bool create) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) - throw Error("cannot open SQLite database '%s'", path); + int flags = SQLITE_OPEN_READWRITE; + if (create) flags |= SQLITE_OPEN_CREATE; + int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + if (ret != SQLITE_OK) { + const char * err = sqlite3_errstr(ret); + throw Error("cannot open SQLite database '%s': %s", path, err); + } if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) SQLiteError::throw_(db, "setting timeout"); + if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") { + // To debug sqlite statements; trace all of them + sqlite3_trace(db, &traceSQL, nullptr); + } + exec("pragma foreign_keys = 1"); } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1d1c553ea..1853731a2 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -98,21 +98,22 @@ struct SQLiteTxn struct SQLiteError : Error { - const char *path; - int errNo, extendedErrNo; + std::string path; + std::string errMsg; + int errNo, extendedErrNo, offset; template<typename... Args> [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { throw_(db, hintfmt(fs, args...)); } - SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf); + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf); protected: template<typename... Args> - SQLiteError(const char *path, int errNo, int extendedErrNo, const std::string & fs, const Args & ... args) - : SQLiteError(path, errNo, extendedErrNo, hintfmt(fs, args...)) + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args) + : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...)) { } [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 62daa838c..a1d4daafd 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -53,8 +53,8 @@ public: { return false; } // FIXME extend daemon protocol, move implementation to RemoteStore - std::optional<std::string> getBuildLog(const StorePath & path) override - { unsupported("getBuildLog"); } + std::optional<std::string> getBuildLogExact(const StorePath & path) override + { unsupported("getBuildLogExact"); } private: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8811ab578..601efa1cc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,21 +19,21 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(const Path & path) const +bool Store::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair<StorePath, Path> Store::toStorePath(const Path & path) const +std::pair<StorePath, Path> Store::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); - Path::size_type slash = path.find('/', storeDir.size() + 1); + auto slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return {parseStorePath(path), ""}; else - return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; + return {parseStorePath(path.substr(0, slash)), (Path) path.substr(slash)}; } @@ -458,6 +458,7 @@ Store::Store(const Params & params) : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { + assertLibStoreInitialized(); } @@ -741,13 +742,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -759,7 +760,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -1209,79 +1210,6 @@ std::string showPaths(const PathSet & paths) } -std::string ValidPathInfo::fingerprint(const Store & store) const -{ - if (narSize == 0) - throw Error("cannot calculate fingerprint of path '%s' because its size is not known", - store.printStorePath(path)); - return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); -} - - -void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint(store))); -} - -bool ValidPathInfo::isContentAddressed(const Store & store) const -{ - if (! ca) return false; - - auto caPath = std::visit(overloaded { - [&](const TextHash & th) { - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](const FixedOutputHash & fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); - - bool res = caPath == path; - - if (!res) - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); - - return res; -} - - -size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const -{ - if (isContentAddressed(store)) return maxSigs; - - size_t good = 0; - for (auto & sig : sigs) - if (checkSignature(store, publicKeys, sig)) - good++; - return good; -} - - -bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const -{ - return verifyDetached(fingerprint(store), sig, publicKeys); -} - - -Strings ValidPathInfo::shortRefs() const -{ - Strings refs; - for (auto & r : references) - refs.push_back(std::string(r.to_string())); - return refs; -} - - Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); @@ -1300,6 +1228,34 @@ Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool req } } +std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path) +{ + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + if (!info->deriver) return std::nullopt; + return *info->deriver; + } catch (InvalidPath &) { + return std::nullopt; + } + } + + if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path)) + return path; + + auto drv = readDerivation(path); + if (!drv.type().hasKnownOutputPaths()) { + // The build log is actually attached to the corresponding + // resolved derivation, so we need to get it first + auto resolvedDrv = drv.tryResolve(*this); + if (resolvedDrv) + return writeDerivation(*this, *resolvedDrv, NoRepair, true); + } + + return path; +} + Derivation Store::readDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, true); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 151ec10d6..4d8db3596 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -71,6 +71,9 @@ class NarInfoDiskCache; class Store; +typedef std::map<std::string, StorePath> OutputPathMap; + + enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; @@ -84,6 +87,8 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -120,6 +125,8 @@ public: typedef std::map<std::string, std::string> Params; + + protected: struct PathInfoCacheValue { @@ -179,7 +186,7 @@ public: /* Return true if ‘path’ is in the Nix store (but not the Nix store itself). */ - bool isInStore(const Path & path) const; + bool isInStore(PathView path) const; /* Return true if ‘path’ is a store path, i.e. a direct child of the Nix store. */ @@ -187,7 +194,7 @@ public: /* Split a path like /nix/store/<hash>-<name>/<bla> into /nix/store/<hash>-<name> and /<bla>. */ - std::pair<StorePath, Path> toStorePath(const Path & path) const; + std::pair<StorePath, Path> toStorePath(PathView path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; @@ -618,6 +625,13 @@ public: */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); + /** + * Given a store path, return the realisation actually used in the realisation of this path: + * - If the path is a content-addressed derivation, try to resolve it + * - Otherwise, find one of its derivers + */ + std::optional<StorePath> getBuildDerivationPath(const StorePath &); + /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() @@ -645,9 +659,6 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - /* * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) @@ -719,6 +730,11 @@ void copyClosure( void removeTempRoots(); +/* Resolve the derived path completely, failing if any derivation output + is unknown. */ +OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); + + /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc new file mode 100644 index 000000000..12be8504d --- /dev/null +++ b/src/libstore/tests/derivation.cc @@ -0,0 +1,124 @@ +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> + +#include "derivations.hh" + +#include "tests/libstore.hh" + +namespace nix { + +class DerivationTest : public LibStoreTest +{ +}; + +#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ + TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ + } + +TEST_JSON(DerivationOutput, inputAddressed, + R"({ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFixed, + R"({ + "hashAlgo": "r:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFloating, + R"({ + "hashAlgo": "r:sha256" + })", + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, deferred, + R"({ })", + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, impure, + R"({ + "hashAlgo": "r:sha256", + "impure": true + })", + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(Derivation, impure, + R"({ + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ + "cat", + "dog" + ] + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + ({ + Derivation drv; + drv.inputSrcs = { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + } + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + drv; + })) + +#undef TEST_JSON + +} diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc new file mode 100644 index 000000000..d1ac2c5e7 --- /dev/null +++ b/src/libstore/tests/derived-path.cc @@ -0,0 +1,62 @@ +#include <regex> + +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace rc { +using namespace nix; + +Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary<StorePath>(), + }); +} + +Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = *gen::arbitrary<StorePath>(), + .outputs = *gen::arbitrary<OutputsSpec>(), + }); +} + +Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 1)) { + case 0: + return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>()); + default: + return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>()); + } +} + +} + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +} diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh new file mode 100644 index 000000000..3bc812440 --- /dev/null +++ b/src/libstore/tests/derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <derived-path.hh> + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<DerivedPath::Opaque> { + static Gen<DerivedPath::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<DerivedPath::Built> { + static Gen<DerivedPath::Built> arbitrary(); +}; + +template<> +struct Arbitrary<DerivedPath> { + static Gen<DerivedPath> arbitrary(); +}; + +} diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh new file mode 100644 index 000000000..05397659b --- /dev/null +++ b/src/libstore/tests/libstore.hh @@ -0,0 +1,23 @@ +#include <gtest/gtest.h> +#include <gmock/gmock.h> + +#include "store-api.hh" + +namespace nix { + +class LibStoreTest : public ::testing::Test { + public: + static void SetUpTestSuite() { + initLibStore(); + } + + protected: + LibStoreTest() + : store(openStore("dummy://")) + { } + + ref<Store> store; +}; + + +} /* namespace nix */ diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index f74295d97..03becc7d1 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -1,6 +1,20 @@ -check: libstore-tests_RUN +check: libstore-tests-exe_RUN -programs += libstore-tests +programs += libstore-tests-exe + +libstore-tests-exe_NAME = libnixstore-tests + +libstore-tests-exe_DIR := $(d) + +libstore-tests-exe_INSTALL_DIR := + +libstore-tests-exe_LIBS = libstore-tests + +libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libstore-tests + +libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) @@ -10,6 +24,6 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil -libstore-tests_LIBS = libstore libutil +libstore-tests_LIBS = libutil-tests libstore libutil -libstore-tests_LDFLAGS := $(GTEST_LIBS) +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/src/libstore/tests/nar-info-disk-cache.cc new file mode 100644 index 000000000..b4bdb8329 --- /dev/null +++ b/src/libstore/tests/nar-info-disk-cache.cc @@ -0,0 +1,123 @@ +#include "nar-info-disk-cache.hh" + +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> +#include "sqlite.hh" +#include <sqlite3.h> + + +namespace nix { + +TEST(NarInfoDiskCacheImpl, create_and_read) { + // This is a large single test to avoid some setup overhead. + + int prio = 12345; + bool wantMassQuery = true; + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir); + Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite"); + + int savedId; + int barId; + SQLite db; + SQLiteStmt getIds; + + { + auto cache = getTestNarInfoDiskCache(dbPath); + + // Set up "background noise" and check that different caches receive different ids + { + auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + ASSERT_NE(bc1, bc2); + barId = bc1; + } + + // Check that the fields are saved and returned correctly. This does not test + // the select statement yet, because of in-memory caching. + savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);; + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(savedId, r->id); + } + + // We're going to pay special attention to the id field because we had a bug + // that changed it. + db = SQLite(dbPath); + getIds.create(db, "select id from BinaryCaches where url = 'http://foo'"); + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + ASSERT_EQ(savedId, q.getInt(0)); + ASSERT_FALSE(q.next()); + } + + // Pretend that the caches are older, but keep one up to date, as "background noise" + db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';"); + + // This shows that the in-memory cache works + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + } + } + + { + // We can't clear the in-memory cache, so we use a new cache object. This is + // more realistic anyway. + auto cache2 = getTestNarInfoDiskCache(dbPath); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_FALSE(r); + } + + // "Update", same data, check that the id number is reused + cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->id, savedId); + } + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + auto currentId = q.getInt(0); + ASSERT_FALSE(q.next()); + ASSERT_EQ(currentId, savedId); + } + + // Check that the fields can be modified, and the id remains the same + { + auto r0 = cache2->upToDateCacheExists("https://bar"); + ASSERT_FALSE(r0); + + cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + auto r = cache2->upToDateCacheExists("https://bar"); + ASSERT_EQ(r->wantMassQuery, !wantMassQuery); + ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->id, barId); + } + + // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp) + // { + // cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20); + // auto r = cache2->upToDateCacheExists("https://bar"); + // ASSERT_EQ(r->wantMassQuery, wantMassQuery); + // ASSERT_EQ(r->priority, prio + 20); + // } + } +} + +} diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc new file mode 100644 index 000000000..984d1d963 --- /dev/null +++ b/src/libstore/tests/outputs-spec.cc @@ -0,0 +1,233 @@ +#include "outputs-spec.hh" + +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +namespace nix { + +#ifndef NDEBUG +TEST(OutputsSpec, no_empty_names) { + ASSERT_DEATH(OutputsSpec::Names { std::set<std::string> { } }, ""); +} +#endif + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(OutputsSpec, bad_ ## NAME) { \ + std::optional OutputsSpecOpt = \ + OutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(OutputsSpecOpt); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") + +#undef TEST_DONT_PARSE + +TEST(OutputsSpec, all) { + std::string_view str = "*"; + OutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out) { + std::string_view str = "out"; + OutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_underscore) { + std::string_view str = "a_b"; + OutputsSpec expected = OutputsSpec::Names { "a_b" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_numberic) { + std::string_view str = "01"; + OutputsSpec expected = OutputsSpec::Names { "01" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out_bin) { + OutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(OutputsSpec::parse("out,bin"), expected); + // N.B. This normalization is OK. + ASSERT_EQ(expected.to_string(), "bin,out"); +} + +#define TEST_SUBSET(X, THIS, THAT) \ + X((OutputsSpec { THIS }).isSubsetOf(THAT)); + +TEST(OutputsSpec, subsets_all_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_names_eq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, subsets_names_names_noneq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" })); +} + +TEST(OutputsSpec, not_subsets_all_names) { + TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, not_subsets_names_names) { + TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" })); +} + +#undef TEST_SUBSET + +#define TEST_UNION(RES, THIS, THAT) \ + ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT)); + +TEST(OutputsSpec, union_all_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_all_names) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, union_names_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_names_names) { + TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" }); +} + +#undef TEST_UNION + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(ExtendedOutputsSpec, bad_ ## NAME) { \ + std::optional extendedOutputsSpecOpt = \ + ExtendedOutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(extendedOutputsSpecOpt); \ + } + +TEST_DONT_PARSE(carot_empty, "^") +TEST_DONT_PARSE(prefix_carot_empty, "foo^") +TEST_DONT_PARSE(garbage, "^&*()") +TEST_DONT_PARSE(double_star, "^**") +TEST_DONT_PARSE(star_first, "^*,foo") +TEST_DONT_PARSE(star_second, "^foo,*") + +#undef TEST_DONT_PARSE + +TEST(ExtendedOutputsSpec, defeault) { + std::string_view str = "foo"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, all) { + std::string_view str = "foo^*"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out) { + std::string_view str = "foo^out"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out_bin) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin"); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out"); +} + +TEST(ExtendedOutputsSpec, many_carrot) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin"); + ASSERT_EQ(prefix, "foo^bar"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out"); +} + + +#define TEST_JSON(TYPE, NAME, STR, VAL) \ + \ + TEST(TYPE, NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + ((nlohmann::json) TYPE { VAL })); \ + } \ + \ + TEST(TYPE, NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + TYPE { VAL }, \ + (STR ## _json).get<TYPE>()); \ + } + +TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { }) +TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" }) +TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" })) + +TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { }) +TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } }) +TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } }) +TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } })) + +#undef TEST_JSON + +} + +namespace rc { +using namespace nix; + +Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 1)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + default: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container<StringSet>(gen::map( + gen::arbitrary<StorePathName>(), + [](StorePathName n) { return n.name; }))), + }); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh new file mode 100644 index 000000000..2d455c817 --- /dev/null +++ b/src/libstore/tests/outputs-spec.hh @@ -0,0 +1,17 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <outputs-spec.hh> + +#include <tests/path.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<OutputsSpec> { + static Gen<OutputsSpec> arbitrary(); +}; + +} diff --git a/src/libstore/tests/path-with-outputs.cc b/src/libstore/tests/path-with-outputs.cc deleted file mode 100644 index 350ea7ffd..000000000 --- a/src/libstore/tests/path-with-outputs.cc +++ /dev/null @@ -1,46 +0,0 @@ -#include "path-with-outputs.hh" - -#include <gtest/gtest.h> - -namespace nix { - -TEST(parseOutputsSpec, basic) -{ - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec)); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^*"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec)); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^out"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin"); - ASSERT_EQ(prefix, "foo^bar"); - ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()"); - ASSERT_EQ(prefix, "foo^&*()"); - ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec)); - } -} - -} diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc new file mode 100644 index 000000000..430aa0099 --- /dev/null +++ b/src/libstore/tests/path.cc @@ -0,0 +1,153 @@ +#include <regex> + +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" + +namespace nix { + +#define STORE_DIR "/nix/store/" +#define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" + +class StorePathTest : public LibStoreTest +{ +}; + +static std::regex nameRegex { std::string { nameRegexStr } }; + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST_F(StorePathTest, bad_ ## NAME) { \ + std::string_view str = \ + STORE_DIR HASH_PART "-" STR; \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + std::string name { STR }; \ + EXPECT_FALSE(std::regex_match(name, nameRegex)); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") +TEST_DONT_PARSE(bang, "foo!o") + +#undef TEST_DONT_PARSE + +#define TEST_DO_PARSE(NAME, STR) \ + TEST_F(StorePathTest, good_ ## NAME) { \ + std::string_view str = \ + STORE_DIR HASH_PART "-" STR; \ + auto p = store->parseStorePath(str); \ + std::string name { p.name() }; \ + EXPECT_TRUE(std::regex_match(name, nameRegex)); \ + } + +// 0-9 a-z A-Z + - . _ ? = + +TEST_DO_PARSE(numbers, "02345") +TEST_DO_PARSE(lower_case, "foo") +TEST_DO_PARSE(upper_case, "FOO") +TEST_DO_PARSE(plus, "foo+bar") +TEST_DO_PARSE(dash, "foo-dev") +TEST_DO_PARSE(underscore, "foo_bar") +TEST_DO_PARSE(period, "foo.txt") +TEST_DO_PARSE(question_mark, "foo?why") +TEST_DO_PARSE(equals_sign, "foo=foo") + +#undef TEST_DO_PARSE + +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os) { + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen<StorePathName> Arbitrary<StorePathName>::arbitrary() +{ + auto len = *gen::inRange<size_t>( + 1, + StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + pre += '.'; + break; + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen<StorePath> Arbitrary<StorePath>::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary<Hash>(), + (*gen::arbitrary<StorePathName>()).name, + }); +} + +} // namespace rc + +namespace nix { + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_regex_accept, + (const StorePath & p)) +{ + RC_ASSERT(std::regex_match(std::string { p.name() }, nameRegex)); +} + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_round_rip, + (const StorePath & p)) +{ + RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); +} + +} diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh new file mode 100644 index 000000000..d7f1a8988 --- /dev/null +++ b/src/libstore/tests/path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <path.hh> + +namespace nix { + +struct StorePathName { + std::string name; +}; + +} + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<StorePathName> { + static Gen<StorePathName> arbitrary(); +}; + +template<> +struct Arbitrary<StorePath> { + static Gen<StorePath> arbitrary(); +}; + +} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4b0636129..0e2b9d12c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -35,10 +35,6 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); -const std::string narVersionMagic1 = "nix-archive-1"; - -static std::string caseHackSuffix = "~nix~case~hack~"; - PathFilter defaultPathFilter = [](const Path &) { return true; }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index ac4183bf5..e42dea540 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -103,7 +103,9 @@ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); -extern const std::string narVersionMagic1; +inline constexpr std::string_view narVersionMagic1 = "nix-archive-1"; + +inline constexpr std::string_view caseHackSuffix = "~nix~case~hack~"; } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 753980fd4..35686a8aa 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -29,7 +29,15 @@ void Args::removeFlag(const std::string & longName) void Completions::add(std::string completion, std::string description) { - assert(description.find('\n') == std::string::npos); + description = trim(description); + // ellipsize overflowing content on the back of the description + auto end_index = description.find_first_of(".\n"); + if (end_index != std::string::npos) { + auto needs_ellipsis = end_index != description.size() - 1; + description.resize(end_index); + if (needs_ellipsis) + description.append(" [...]"); + } insert(Completion { .completion = completion, .description = description @@ -324,7 +332,7 @@ MultiCommand::MultiCommand(const Commands & commands_) expectArgs({ .label = "subcommand", .optional = true, - .handler = {[=](std::string s) { + .handler = {[=,this](std::string s) { assert(!command); auto i = commands.find(s); if (i == commands.end()) { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc new file mode 100644 index 000000000..b132b4262 --- /dev/null +++ b/src/libutil/canon-path.cc @@ -0,0 +1,103 @@ +#include "canon-path.hh" +#include "util.hh" + +namespace nix { + +CanonPath CanonPath::root = CanonPath("/"); + +CanonPath::CanonPath(std::string_view raw) + : path(absPath((Path) raw, "/")) +{ } + +CanonPath::CanonPath(std::string_view raw, const CanonPath & root) + : path(absPath((Path) raw, root.abs())) +{ } + +std::optional<CanonPath> CanonPath::parent() const +{ + if (isRoot()) return std::nullopt; + return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/')))); +} + +void CanonPath::pop() +{ + assert(!isRoot()); + path.resize(std::max((size_t) 1, path.rfind('/'))); +} + +bool CanonPath::isWithin(const CanonPath & parent) const +{ + return !( + path.size() < parent.path.size() + || path.substr(0, parent.path.size()) != parent.path + || (parent.path.size() > 1 && path.size() > parent.path.size() + && path[parent.path.size()] != '/')); +} + +CanonPath CanonPath::removePrefix(const CanonPath & prefix) const +{ + assert(isWithin(prefix)); + if (prefix.isRoot()) return *this; + if (path.size() == prefix.path.size()) return root; + return CanonPath(unchecked_t(), path.substr(prefix.path.size())); +} + +void CanonPath::extend(const CanonPath & x) +{ + if (x.isRoot()) return; + if (isRoot()) + path += x.rel(); + else + path += x.abs(); +} + +CanonPath CanonPath::operator + (const CanonPath & x) const +{ + auto res = *this; + res.extend(x); + return res; +} + +void CanonPath::push(std::string_view c) +{ + assert(c.find('/') == c.npos); + assert(c != "." && c != ".."); + if (!isRoot()) path += '/'; + path += c; +} + +CanonPath CanonPath::operator + (std::string_view c) const +{ + auto res = *this; + res.push(c); + return res; +} + +bool CanonPath::isAllowed(const std::set<CanonPath> & allowed) const +{ + /* Check if `this` is an exact match or the parent of an + allowed path. */ + auto lb = allowed.lower_bound(*this); + if (lb != allowed.end()) { + if (lb->isWithin(*this)) + return true; + } + + /* Check if a parent of `this` is allowed. */ + auto path = *this; + while (!path.isRoot()) { + path.pop(); + if (allowed.count(path)) + return true; + } + + return false; +} + +std::ostream & operator << (std::ostream & stream, const CanonPath & path) +{ + stream << path.abs(); + return stream; +} + +} diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh new file mode 100644 index 000000000..9d5984584 --- /dev/null +++ b/src/libutil/canon-path.hh @@ -0,0 +1,173 @@ +#pragma once + +#include <string> +#include <optional> +#include <cassert> +#include <iostream> +#include <set> + +namespace nix { + +/* A canonical representation of a path. It ensures the following: + + - It always starts with a slash. + + - It never ends with a slash, except if the path is "/". + + - A slash is never followed by a slash (i.e. no empty components). + + - There are no components equal to '.' or '..'. + + Note that the path does not need to correspond to an actually + existing path, and there is no guarantee that symlinks are + resolved. +*/ +class CanonPath +{ + std::string path; + +public: + + /* Construct a canon path from a non-canonical path. Any '.', '..' + or empty components are removed. */ + CanonPath(std::string_view raw); + + explicit CanonPath(const char * raw) + : CanonPath(std::string_view(raw)) + { } + + struct unchecked_t { }; + + CanonPath(unchecked_t _, std::string path) + : path(std::move(path)) + { } + + static CanonPath root; + + /* If `raw` starts with a slash, return + `CanonPath(raw)`. Otherwise return a `CanonPath` representing + `root + "/" + raw`. */ + CanonPath(std::string_view raw, const CanonPath & root); + + bool isRoot() const + { return path.size() <= 1; } + + explicit operator std::string_view() const + { return path; } + + const std::string & abs() const + { return path; } + + /* Like abs(), but return an empty string if this path is + '/'. Thus the returned string never ends in a slash. */ + const std::string & absOrEmpty() const + { + const static std::string epsilon; + return isRoot() ? epsilon : path; + } + + const char * c_str() const + { return path.c_str(); } + + std::string_view rel() const + { return ((std::string_view) path).substr(1); } + + struct Iterator + { + std::string_view remaining; + size_t slash; + + Iterator(std::string_view remaining) + : remaining(remaining) + , slash(remaining.find('/')) + { } + + bool operator != (const Iterator & x) const + { return remaining.data() != x.remaining.data(); } + + const std::string_view operator * () const + { return remaining.substr(0, slash); } + + void operator ++ () + { + if (slash == remaining.npos) + remaining = remaining.substr(remaining.size()); + else { + remaining = remaining.substr(slash + 1); + slash = remaining.find('/'); + } + } + }; + + Iterator begin() const { return Iterator(rel()); } + Iterator end() const { return Iterator(rel().substr(path.size() - 1)); } + + std::optional<CanonPath> parent() const; + + /* Remove the last component. Panics if this path is the root. */ + void pop(); + + std::optional<std::string_view> dirOf() const + { + if (isRoot()) return std::nullopt; + return ((std::string_view) path).substr(0, path.rfind('/')); + } + + std::optional<std::string_view> baseName() const + { + if (isRoot()) return std::nullopt; + return ((std::string_view) path).substr(path.rfind('/') + 1); + } + + bool operator == (const CanonPath & x) const + { return path == x.path; } + + bool operator != (const CanonPath & x) const + { return path != x.path; } + + /* Compare paths lexicographically except that path separators + are sorted before any other character. That is, in the sorted order + a directory is always followed directly by its children. For + instance, 'foo' < 'foo/bar' < 'foo!'. */ + bool operator < (const CanonPath & x) const + { + auto i = path.begin(); + auto j = x.path.begin(); + for ( ; i != path.end() && j != x.path.end(); ++i, ++j) { + auto c_i = *i; + if (c_i == '/') c_i = 0; + auto c_j = *j; + if (c_j == '/') c_j = 0; + if (c_i < c_j) return true; + if (c_i > c_j) return false; + } + return i == path.end() && j != x.path.end(); + } + + /* Return true if `this` is equal to `parent` or a child of + `parent`. */ + bool isWithin(const CanonPath & parent) const; + + CanonPath removePrefix(const CanonPath & prefix) const; + + /* Append another path to this one. */ + void extend(const CanonPath & x); + + /* Concatenate two paths. */ + CanonPath operator + (const CanonPath & x) const; + + /* Add a path component to this one. It must not contain any slashes. */ + void push(std::string_view c); + + CanonPath operator + (std::string_view c) const; + + /* Check whether access to this path is allowed, which is the case + if 1) `this` is within any of the `allowed` paths; or 2) any of + the `allowed` paths are within `this`. (The latter condition + ensures access to the parents of allowed paths.) */ + bool isAllowed(const std::set<CanonPath> & allowed) const; +}; + +std::ostream & operator << (std::ostream & stream, const CanonPath & path); + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 9bb412b4f..b349f2d80 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -209,7 +209,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s); }}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -218,7 +218,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s, true); }}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, }); } @@ -270,13 +270,13 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(true); }} + .handler = {[this]() { override(true); }} }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(false); }} + .handler = {[this]() { override(false); }} }); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 79ec0f9cf..7ac43c854 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -250,11 +250,15 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } + template<typename U> + bool operator ==(const U & v2) const { return value == v2; } + template<typename U> + bool operator !=(const U & v2) const { return value != v2; } + template<typename U> + void operator =(const U & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void setDefault(const T & v) { if (!overridden) value = v; } + template<typename U> + void setDefault(const U & v) { if (!overridden) value = v; } void set(const std::string & str, bool append = false) override; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 9172f67a6..e4f0d4677 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -9,9 +9,9 @@ namespace nix { const std::string nativeSystem = SYSTEM; -void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) +void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame) { - err.traces.push_front(Trace { .pos = e, .hint = hint }); + err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } // c++ std::exception descendants must have a 'const char* what()' function. @@ -30,91 +30,46 @@ const std::string & BaseError::calcWhat() const std::optional<std::string> ErrorInfo::programName = std::nullopt; -std::ostream & operator<<(std::ostream & os, const hintformat & hf) +std::ostream & operator <<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -std::string showErrPos(const ErrPos & errPos) +std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) { - if (errPos.line > 0) { - if (errPos.column > 0) { - return fmt("%d:%d", errPos.line, errPos.column); - } else { - return fmt("%d", errPos.line); - } - } - else { - return ""; - } + pos.print(str); + str << ":" << pos.line; + if (pos.column > 0) + str << ":" << pos.column; + return str; } -std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) +std::optional<LinesOfCode> AbstractPos::getCodeLines() const { - if (errPos.line <= 0) + if (line == 0) return std::nullopt; - if (errPos.origin == foFile) { - LinesOfCode loc; - try { - // FIXME: when running as the daemon, make sure we don't - // open a file to which the client doesn't have access. - AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) return {}; - - // count the newlines. - int count = 0; - std::string line; - int pl = errPos.line - 1; - do - { - line = readLine(fd.get()); - ++count; - if (count < pl) - ; - else if (count == pl) - loc.prevLineOfCode = line; - else if (count == pl + 1) - loc.errLineOfCode = line; - else if (count == pl + 2) { - loc.nextLineOfCode = line; - break; - } - } while (true); - return loc; - } - catch (EndOfFile & eof) { - if (loc.errLineOfCode.has_value()) - return loc; - else - return std::nullopt; - } - catch (std::exception & e) { - return std::nullopt; - } - } else { - std::istringstream iss(errPos.file); + if (auto source = getSource()) { + + std::istringstream iss(*source); // count the newlines. int count = 0; - std::string line; - int pl = errPos.line - 1; + std::string curLine; + int pl = line - 1; LinesOfCode loc; - do - { - std::getline(iss, line); + do { + std::getline(iss, curLine); ++count; if (count < pl) - { ; - } else if (count == pl) { - loc.prevLineOfCode = line; + loc.prevLineOfCode = curLine; } else if (count == pl + 1) { - loc.errLineOfCode = line; + loc.errLineOfCode = curLine; } else if (count == pl + 2) { - loc.nextLineOfCode = line; + loc.nextLineOfCode = curLine; break; } @@ -124,12 +79,14 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) return loc; } + + return std::nullopt; } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, const std::string & prefix, - const ErrPos & errPos, + const AbstractPos & errPos, const LinesOfCode & loc) { // previous line of code. @@ -176,28 +133,6 @@ void printCodeLines(std::ostream & out, } } -void printAtPos(const ErrPos & pos, std::ostream & out) -{ - if (pos) { - switch (pos.origin) { - case foFile: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); - break; - } - case foString: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); - break; - } - case foStdin: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); - break; - } - default: - throw Error("invalid FileOrigin in errPos"); - } - } -} - static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s) { std::string res; @@ -262,49 +197,160 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s prefix += ":" ANSI_NORMAL " "; std::ostringstream oss; - oss << einfo.msg << "\n"; - if (einfo.errPos.has_value() && *einfo.errPos) { - oss << "\n"; - printAtPos(*einfo.errPos, oss); + auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; + + /* + * Traces + * ------ + * + * The semantics of traces is a bit weird. We have only one option to + * print them and to make them verbose (--show-trace). In the code they + * are always collected, but they are not printed by default. The code + * also collects more traces when the option is on. This means that there + * is no way to print the simplified traces at all. + * + * I (layus) designed the code to attach positions to a restricted set of + * messages. This means that we have a lot of traces with no position at + * all, including most of the base error messages. For example "type + * error: found a string while a set was expected" has no position, but + * will come with several traces detailing it's precise relation to the + * closest know position. This makes erroring without printing traces + * quite useless. + * + * This is why I introduced the idea to always print a few traces on + * error. The number 3 is quite arbitrary, and was selected so as not to + * clutter the console on error. For the same reason, a trace with an + * error position takes more space, and counts as two traces towards the + * limit. + * + * The rest is truncated, unless --show-trace is passed. This preserves + * the same bad semantics of --show-trace to both show the trace and + * augment it with new data. Not too sure what is the best course of + * action. + * + * The issue is that it is fundamentally hard to provide a trace for a + * lazy language. The trace will only cover the current spine of the + * evaluation, missing things that have been evaluated before. For + * example, most type errors are hard to inspect because there is not + * trace for the faulty value. These errors should really print the faulty + * value itself. + * + * In function calls, the --show-trace flag triggers extra traces for each + * function invocation. These work as scopes, allowing to follow the + * current spine of the evaluation graph. Without that flag, the error + * trace should restrict itself to a restricted prefix of that trace, + * until the first scope. If we ever get to such a precise error + * reporting, there would be no need to add an arbitrary limit here. We + * could always print the full trace, and it would just be small without + * the flag. + * + * One idea I had is for XxxError.addTrace() to perform nothing if one + * scope has already been traced. Alternatively, we could stop here when + * we encounter such a scope instead of after an arbitrary number of + * traces. This however requires to augment traces with the notion of + * "scope". + * + * This is particularly visible in code like evalAttrs(...) where we have + * to make a decision between the two following options. + * + * ``` long traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) + * { + * try { + * e->eval(*this, env, v); + * if (v.type() != nAttrs) + * throwTypeError("value is %1% while a set was expected", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * ``` short traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) + * { + * e->eval(*this, env, v); + * try { + * if (v.type() != nAttrs) + * throwTypeError("value is %1% while a set was expected", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * The second example can be rewritten more concisely, but kept in this + * form to highlight the symmetry. The first option adds more information, + * because whatever caused an error down the line, in the generic eval + * function, will get annotated with the code location that uses and + * required it. The second option is less verbose, but does not provide + * any context at all as to where and why a failing value was required. + * + * Scopes would fix that, by adding context only when --show-trace is + * passed, and keeping the trace terse otherwise. + * + */ + + // Enough indent to align with with the `... ` + // prepended to each element of the trace + auto ellipsisIndent = " "; + + bool frameOnly = false; + if (!einfo.traces.empty()) { + size_t count = 0; + for (const auto & trace : einfo.traces) { + if (!showTrace && count > 3) { + oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; + break; + } + + if (trace.hint.str().empty()) continue; + if (frameOnly && !trace.frame) continue; + + count++; + frameOnly = trace.frame; + + oss << "\n" << "… " << trace.hint.str() << "\n"; + + if (trace.pos) { + count++; + + oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; + + if (auto loc = trace.pos->getCodeLines()) { + oss << "\n"; + printCodeLines(oss, "", *trace.pos, *loc); + oss << "\n"; + } else + oss << noSource; + } + } + oss << "\n" << prefix; + } + + oss << einfo.msg << "\n"; - auto loc = getCodeLines(*einfo.errPos); + if (einfo.errPos) { + oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - // lines of code. - if (loc.has_value()) { + if (auto loc = einfo.errPos->getCodeLines()) { oss << "\n"; printCodeLines(oss, "", *einfo.errPos, *loc); oss << "\n"; - } + } else + oss << noSource; } auto suggestions = einfo.suggestions.trim(); - if (! suggestions.suggestions.empty()){ + if (!suggestions.suggestions.empty()) { oss << "Did you mean " << suggestions.trim() << "?" << std::endl; } - // traces - if (showTrace && !einfo.traces.empty()) { - for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { - oss << "\n" << "… " << iter->hint.str() << "\n"; - - if (iter->pos.has_value() && (*iter->pos)) { - auto pos = iter->pos.value(); - oss << "\n"; - printAtPos(pos, oss); - - auto loc = getCodeLines(pos); - if (loc.has_value()) { - oss << "\n"; - printCodeLines(oss, "", pos, *loc); - oss << "\n"; - } - } - } - } - out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str())); return out; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 3d1479c54..0ebeaba61 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -54,13 +54,6 @@ typedef enum { lvlVomit } Verbosity; -/* adjust Pos::origin bit width when adding stuff here */ -typedef enum { - foFile, - foStdin, - foString -} FileOrigin; - // the lines of code surrounding an error. struct LinesOfCode { std::optional<std::string> prevLineOfCode; @@ -68,54 +61,40 @@ struct LinesOfCode { std::optional<std::string> nextLineOfCode; }; -// ErrPos indicates the location of an error in a nix file. -struct ErrPos { - int line = 0; - int column = 0; - std::string file; - FileOrigin origin; +/* An abstract type that represents a location in a source file. */ +struct AbstractPos +{ + uint32_t line = 0; + uint32_t column = 0; - operator bool() const - { - return line != 0; - } + /* Return the contents of the source file. */ + virtual std::optional<std::string> getSource() const + { return std::nullopt; }; - // convert from the Pos struct, found in libexpr. - template <class P> - ErrPos & operator=(const P & pos) - { - origin = pos.origin; - line = pos.line; - column = pos.column; - file = pos.file; - return *this; - } + virtual void print(std::ostream & out) const = 0; - template <class P> - ErrPos(const P & p) - { - *this = p; - } + std::optional<LinesOfCode> getCodeLines() const; + + virtual ~AbstractPos() = default; }; -std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos); +std::ostream & operator << (std::ostream & str, const AbstractPos & pos); void printCodeLines(std::ostream & out, const std::string & prefix, - const ErrPos & errPos, + const AbstractPos & errPos, const LinesOfCode & loc); -void printAtPos(const ErrPos & pos, std::ostream & out); - struct Trace { - std::optional<ErrPos> pos; + std::shared_ptr<AbstractPos> pos; hintformat hint; + bool frame; }; struct ErrorInfo { Verbosity level; hintformat msg; - std::optional<ErrPos> errPos; + std::shared_ptr<AbstractPos> errPos; std::list<Trace> traces; Suggestions suggestions; @@ -138,6 +117,8 @@ protected: public: unsigned int status = 1; // exit status + BaseError(const BaseError &) = default; + template<typename... Args> BaseError(unsigned int status, const Args & ... args) : err { .level = lvlError, .msg = hintfmt(args...) } @@ -176,15 +157,22 @@ public: const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } + void pushTrace(Trace trace) + { + err.traces.push_front(trace); + } + template<typename... Args> - void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) + void addTrace(std::shared_ptr<AbstractPos> && e, std::string_view fs, const Args & ... args) { - addTrace(e, hintfmt(fs, args...)); + addTrace(std::move(e), hintfmt(std::string(fs), args...)); } - void addTrace(std::optional<ErrPos> e, hintformat hint); + void addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame = false); bool hasTrace() const { return !err.traces.empty(); } + + const ErrorInfo & info() { return err; }; }; #define MakeError(newClass, superClass) \ diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..58d762ebb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,6 +16,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, { Xp::Cgroups, "cgroups" }, + { Xp::DiscardReferences, "discard-references" }, }; const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..ac372e03e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -25,6 +25,7 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, + DiscardReferences, }; /** diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 7664e5c04..e879fd3b8 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -148,7 +148,7 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) return f; } -inline hintformat hintfmt(std::string plain_string) +inline hintformat hintfmt(const std::string & plain_string) { // we won't be receiving any args in this case, so just print the original string return hintfmt("%s", normaltxt(plain_string)); diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh new file mode 100644 index 000000000..bd75748ad --- /dev/null +++ b/src/libutil/json-impls.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "nlohmann/json_fwd.hpp" + +// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types +#define JSON_IMPL(TYPE) \ + namespace nlohmann { \ + using namespace nix; \ + template <> \ + struct adl_serializer<TYPE> { \ + static TYPE from_json(const json & json); \ + static void to_json(json & json, TYPE t); \ + }; \ + } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index cb2b15b41..904ba6ebe 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -105,14 +105,6 @@ public: Verbosity verbosity = lvlInfo; -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - warn(fs.s); - haveWarned = true; - } -} - void writeToStderr(std::string_view s) { try { @@ -130,15 +122,30 @@ Logger * makeSimpleLogger(bool printBuildLogs) return new SimpleLogger(printBuildLogs); } -std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32}; +std::atomic<uint64_t> nextId{0}; Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++) + : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32)) { logger.startActivity(id, lvl, type, s, fields, parent); } +void to_json(nlohmann::json & json, std::shared_ptr<AbstractPos> pos) +{ + if (pos) { + json["line"] = pos->line; + json["column"] = pos->column; + std::ostringstream str; + pos->print(str); + json["file"] = str.str(); + } else { + json["line"] = nullptr; + json["column"] = nullptr; + json["file"] = nullptr; + } +} + struct JSONLogger : Logger { Logger & prevLogger; @@ -185,27 +192,14 @@ struct JSONLogger : Logger { json["level"] = ei.level; json["msg"] = oss.str(); json["raw_msg"] = ei.msg.str(); - - if (ei.errPos.has_value() && (*ei.errPos)) { - json["line"] = ei.errPos->line; - json["column"] = ei.errPos->column; - json["file"] = ei.errPos->file; - } else { - json["line"] = nullptr; - json["column"] = nullptr; - json["file"] = nullptr; - } + to_json(json, ei.errPos); if (loggerSettings.showTrace.get() && !ei.traces.empty()) { nlohmann::json traces = nlohmann::json::array(); for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) { nlohmann::json stackFrame; stackFrame["raw_msg"] = iter->hint.str(); - if (iter->pos.has_value() && (*iter->pos)) { - stackFrame["line"] = iter->pos->line; - stackFrame["column"] = iter->pos->column; - stackFrame["file"] = iter->pos->file; - } + to_json(stackFrame, iter->pos); traces.push_back(stackFrame); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index d0817b4a9..4642c49f7 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -82,7 +82,7 @@ public: log(lvlInfo, fs); } - virtual void logEI(const ErrorInfo &ei) = 0; + virtual void logEI(const ErrorInfo & ei) = 0; void logEI(Verbosity lvl, ErrorInfo ei) { @@ -225,7 +225,11 @@ inline void warn(const std::string & fs, const Args & ... args) logger->warn(f.str()); } -void warnOnce(bool & haveWarned, const FormatOrString & fs); +#define warnOnce(haveWarned, args...) \ + if (!haveWarned) { \ + haveWarned = true; \ + warn(args); \ + } void writeToStderr(std::string_view s); diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 5ee0b88ef..9518cf8aa 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -22,27 +22,38 @@ public: { thread = std::thread([fd]() { while (true) { - /* Wait indefinitely until a POLLHUP occurs. */ - struct pollfd fds[1]; - fds[0].fd = fd; - /* This shouldn't be necessary, but macOS doesn't seem to - like a zeroed out events field. - See rdar://37537852. - */ - fds[0].events = POLLHUP; - auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen - /* This shouldn't happen, but can on macOS due to a bug. - See rdar://37550628. - - This may eventually need a delay or further - coordination with the main thread if spinning proves - too harmful. - */ - if (count == 0) continue; - assert(fds[0].revents & POLLHUP); - triggerInterrupt(); - break; + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* Polling for no specific events (i.e. just waiting + for an error/hangup) doesn't work on macOS + anymore. So wait for read events and ignore + them. */ + fds[0].events = + #ifdef __APPLE__ + POLLRDNORM + #else + 0 + #endif + ; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + if (fds[0].revents & POLLHUP) { + triggerInterrupt(); + break; + } + /* This will only happen on macOS. We sleep a bit to + avoid waking up too often if the client is sending + input. */ + sleep(1); } }); }; diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc new file mode 100644 index 000000000..f66accb10 --- /dev/null +++ b/src/libutil/namespaces.cc @@ -0,0 +1,97 @@ +#if __linux__ + +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" + +#include <sys/mount.h> + +namespace nix { + +bool userNamespacesSupported() +{ + static auto res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + try { + Pid pid = startProcess([&]() + { + _exit(0); + }, { + .cloneFlags = CLONE_NEWUSER + }); + + auto r = pid.wait(); + assert(!r); + } catch (SysError & e) { + debug("user namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +bool mountAndPidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + try { + + Pid pid = startProcess([&]() + { + /* Make sure we don't remount the parent's /proc. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + _exit(1); + + /* Test whether we can remount /proc. The kernel disallows + this if /proc is not fully visible, i.e. if there are + filesystems mounted on top of files inside /proc. See + https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + if (mount("none", "/proc", "proc", 0, 0) == -1) + _exit(2); + + _exit(0); + }, { + .cloneFlags = CLONE_NEWNS | CLONE_NEWPID | (userNamespacesSupported() ? CLONE_NEWUSER : 0) + }); + + if (pid.wait()) { + debug("PID namespaces do not work on this system: cannot remount /proc"); + return false; + } + + } catch (SysError & e) { + debug("mount namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +} + +#endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh new file mode 100644 index 000000000..e82379b9c --- /dev/null +++ b/src/libutil/namespaces.hh @@ -0,0 +1,13 @@ +#pragma once + +namespace nix { + +#if __linux__ + +bool userNamespacesSupported(); + +bool mountAndPidNamespacesSupported(); + +#endif + +} diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index bf26321db..7d38b059c 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -83,6 +83,11 @@ public: return p != other.p; } + bool operator < (const ref<T> & other) const + { + return p < other.p; + } + private: template<typename T2, typename... Args> diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh new file mode 100644 index 000000000..0b997b25a --- /dev/null +++ b/src/libutil/regex-combinators.hh @@ -0,0 +1,30 @@ +#pragma once + +#include <string_view> + +namespace nix::regex { + +// TODO use constexpr string building like +// https://github.com/akrzemi1/static_string/blob/master/include/ak_toolkit/static_string.hpp + +static inline std::string either(std::string_view a, std::string_view b) +{ + return std::string { a } + "|" + b; +} + +static inline std::string group(std::string_view a) +{ + return std::string { "(" } + a + ")"; +} + +static inline std::string many(std::string_view a) +{ + return std::string { "(?:" } + a + ")*"; +} + +static inline std::string list(std::string_view a) +{ + return std::string { a } + many(group("," + a)); +} + +} diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 2c3597775..c653db9d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -338,7 +338,7 @@ Sink & operator << (Sink & sink, const StringSet & s) Sink & operator << (Sink & sink, const Error & ex) { - auto info = ex.info(); + auto & info = ex.info(); sink << "Error" << info.level diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 84847835a..7da5b07fd 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -331,17 +331,9 @@ T readNum(Source & source) unsigned char buf[8]; source((char *) buf, sizeof(buf)); - uint64_t n = - ((uint64_t) buf[0]) | - ((uint64_t) buf[1] << 8) | - ((uint64_t) buf[2] << 16) | - ((uint64_t) buf[3] << 24) | - ((uint64_t) buf[4] << 32) | - ((uint64_t) buf[5] << 40) | - ((uint64_t) buf[6] << 48) | - ((uint64_t) buf[7] << 56); - - if (n > (uint64_t)std::numeric_limits<T>::max()) + auto n = readLittleEndian<uint64_t>(buf); + + if (n > (uint64_t) std::numeric_limits<T>::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; diff --git a/src/libutil/tests/canon-path.cc b/src/libutil/tests/canon-path.cc new file mode 100644 index 000000000..c1c5adadf --- /dev/null +++ b/src/libutil/tests/canon-path.cc @@ -0,0 +1,155 @@ +#include "canon-path.hh" + +#include <gtest/gtest.h> + +namespace nix { + + TEST(CanonPath, basic) { + { + CanonPath p("/"); + ASSERT_EQ(p.abs(), "/"); + ASSERT_EQ(p.rel(), ""); + ASSERT_EQ(p.baseName(), std::nullopt); + ASSERT_EQ(p.dirOf(), std::nullopt); + ASSERT_FALSE(p.parent()); + } + + { + CanonPath p("/foo//"); + ASSERT_EQ(p.abs(), "/foo"); + ASSERT_EQ(p.rel(), "foo"); + ASSERT_EQ(*p.baseName(), "foo"); + ASSERT_EQ(*p.dirOf(), ""); // FIXME: do we want this? + ASSERT_EQ(p.parent()->abs(), "/"); + } + + { + CanonPath p("foo/bar"); + ASSERT_EQ(p.abs(), "/foo/bar"); + ASSERT_EQ(p.rel(), "foo/bar"); + ASSERT_EQ(*p.baseName(), "bar"); + ASSERT_EQ(*p.dirOf(), "/foo"); + ASSERT_EQ(p.parent()->abs(), "/foo"); + } + + { + CanonPath p("foo//bar/"); + ASSERT_EQ(p.abs(), "/foo/bar"); + ASSERT_EQ(p.rel(), "foo/bar"); + ASSERT_EQ(*p.baseName(), "bar"); + ASSERT_EQ(*p.dirOf(), "/foo"); + } + } + + TEST(CanonPath, pop) { + CanonPath p("foo/bar/x"); + ASSERT_EQ(p.abs(), "/foo/bar/x"); + p.pop(); + ASSERT_EQ(p.abs(), "/foo/bar"); + p.pop(); + ASSERT_EQ(p.abs(), "/foo"); + p.pop(); + ASSERT_EQ(p.abs(), "/"); + } + + TEST(CanonPath, removePrefix) { + CanonPath p1("foo/bar"); + CanonPath p2("foo/bar/a/b/c"); + ASSERT_EQ(p2.removePrefix(p1).abs(), "/a/b/c"); + ASSERT_EQ(p1.removePrefix(p1).abs(), "/"); + ASSERT_EQ(p1.removePrefix(CanonPath("/")).abs(), "/foo/bar"); + } + + TEST(CanonPath, iter) { + { + CanonPath p("a//foo/bar//"); + std::vector<std::string_view> ss; + for (auto & c : p) ss.push_back(c); + ASSERT_EQ(ss, std::vector<std::string_view>({"a", "foo", "bar"})); + } + + { + CanonPath p("/"); + std::vector<std::string_view> ss; + for (auto & c : p) ss.push_back(c); + ASSERT_EQ(ss, std::vector<std::string_view>()); + } + } + + TEST(CanonPath, concat) { + { + CanonPath p1("a//foo/bar//"); + CanonPath p2("xyzzy/bla"); + ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla"); + } + + { + CanonPath p1("/"); + CanonPath p2("/a/b"); + ASSERT_EQ((p1 + p2).abs(), "/a/b"); + } + + { + CanonPath p1("/a/b"); + CanonPath p2("/"); + ASSERT_EQ((p1 + p2).abs(), "/a/b"); + } + + { + CanonPath p("/foo/bar"); + ASSERT_EQ((p + "x").abs(), "/foo/bar/x"); + } + + { + CanonPath p("/"); + ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar"); + } + } + + TEST(CanonPath, within) { + { + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); + } + } + + TEST(CanonPath, sort) { + ASSERT_FALSE(CanonPath("foo") < CanonPath("foo")); + ASSERT_TRUE (CanonPath("foo") < CanonPath("foo/bar")); + ASSERT_TRUE (CanonPath("foo/bar") < CanonPath("foo!")); + ASSERT_FALSE(CanonPath("foo!") < CanonPath("foo")); + ASSERT_TRUE (CanonPath("foo") < CanonPath("foo!")); + } + + TEST(CanonPath, allowed) { + { + std::set<CanonPath> allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; + + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + } +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 412c03030..e4e928b3b 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -1,5 +1,12 @@ -#include "hash.hh" +#include <regex> + +#include <nlohmann/json.hpp> #include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include <hash.hh> + +#include "tests/hash.hh" namespace nix { @@ -73,3 +80,16 @@ namespace nix { "c7d329eeb6dd26545e96e55b874be909"); } } + +namespace rc { +using namespace nix; + +Gen<Hash> Arbitrary<Hash>::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary<uint8_t>(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh new file mode 100644 index 000000000..9e9650e6e --- /dev/null +++ b/src/libutil/tests/hash.hh @@ -0,0 +1,15 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <hash.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<Hash> { + static Gen<Hash> arbitrary(); +}; + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 815e18560..167915439 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -2,14 +2,28 @@ check: libutil-tests_RUN programs += libutil-tests +libutil-tests-exe_NAME = libnixutil-tests + +libutil-tests-exe_DIR := $(d) + +libutil-tests-exe_INSTALL_DIR := + +libutil-tests-exe_LIBS = libutil-tests + +libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libutil-tests + +libutil-tests_NAME = libnixutil-tests + libutil-tests_DIR := $(d) libutil-tests_INSTALL_DIR := libutil-tests_SOURCES := $(wildcard $(d)/*.cc) -libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr +libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $(GTEST_LIBS) +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 6e325db98..250e83a38 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -312,6 +312,42 @@ namespace nix { } /* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + + TEST(getLine, all) { + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } + } + + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index c3b233797..e0c438b4d 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -99,6 +99,27 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5b7abeb49..4e43455e1 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url) auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f2caaa40..885bae69c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -36,6 +36,7 @@ #ifdef __linux__ #include <sys/prctl.h> #include <sys/resource.h> +#include <sys/mman.h> #include <cmath> #endif @@ -537,6 +538,16 @@ std::string getUserName() return name; } +Path getHomeOf(uid_t userId) +{ + std::vector<char> buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} Path getHome() { @@ -558,13 +569,7 @@ Path getHome() } } if (!homeDir) { - std::vector<char> buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; + homeDir = getHomeOf(geteuid()); if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); } @@ -604,6 +609,19 @@ Path getDataDir() return dataDir ? *dataDir : getHome() + "/.local/share"; } +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + std::optional<Path> getSelfExe() { @@ -1047,9 +1065,17 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) } +static int childEntry(void * arg) +{ + auto main = (std::function<void()> *) arg; + (*main)(); + return 1; +} + + pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { - auto wrapper = [&]() { + std::function<void()> wrapper = [&]() { if (!options.allowVfork) logger = makeSimpleLogger(); try { @@ -1069,7 +1095,27 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) _exit(1); }; - pid_t pid = doFork(options.allowVfork, wrapper); + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); return pid; @@ -1594,6 +1640,21 @@ std::string stripIndentation(std::string_view s) } +std::pair<std::string_view, std::string_view> getLine(std::string_view s) +{ + auto newline = s.find('\n'); + + if (newline == s.npos) { + return {s, ""}; + } else { + auto line = s.substr(0, newline); + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + return {line, s.substr(newline + 1)}; + } +} + + ////////////////////////////////////////////////////////////////////// static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 94d8cc555..b5625ecef 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -137,6 +137,9 @@ void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); +/* Return the given user's home directory from /etc/passwd. */ +Path getHomeOf(uid_t userId); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); @@ -155,6 +158,12 @@ Path getDataDir(); /* Return the path of the current executable. */ std::optional<Path> getSelfExe(); +/* Return $XDG_STATE_HOME or $HOME/.local/state. */ +Path getStateDir(); + +/* Create the Nix state directory and return the path to it. */ +Path createNixStateDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -298,6 +307,7 @@ struct ProcessOptions bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; + int cloneFlags = 0; // use clone() with the specified flags (Linux only) }; pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); @@ -510,6 +520,18 @@ std::optional<N> string2Float(const std::string_view s) } +/* Convert a little-endian integer to host order. */ +template<typename T> +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i, ++p) { + x |= ((T) *p) << (i * 8); + } + return x; +} + + /* Return true iff `s' starts with `prefix'. */ bool hasPrefix(std::string_view s, std::string_view prefix); @@ -563,6 +585,12 @@ std::string base64Decode(std::string_view s); std::string stripIndentation(std::string_view s); +/* Get the prefix of 's' up to and excluding the next line break (LF + optionally preceded by CR), and the remainder following the line + break. */ +std::pair<std::string_view, std::string_view> getLine(std::string_view s); + + /* Get a value for the specified key from an associate container. */ template <class T> const typename T::mapped_type * get(const T & map, const typename T::key_type & key) @@ -737,4 +765,13 @@ inline std::string operator + (std::string && s, std::string_view s2) return std::move(s); } +inline std::string operator + (std::string_view s1, const char * s2) +{ + std::string s; + s.reserve(s1.size() + strlen(s2)); + s.append(s1); + s.append(s2); + return s; +} + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index adcaab686..da76c2ace 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -397,7 +397,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { .drvPath = bashDrv, - .outputs = {"out"}, + .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -421,7 +421,7 @@ static void main_nix_build(int argc, char * * argv) { pathsToBuild.push_back(DerivedPath::Built { .drvPath = inputDrv, - .outputs = inputOutputs + .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); } @@ -536,7 +536,9 @@ static void main_nix_build(int argc, char * * argv) "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " @@ -591,7 +593,7 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index cf52b03b4..338a7d18e 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "util.hh" #include <fcntl.h> #include <regex> @@ -162,11 +164,11 @@ static int main_nix_channel(int argc, char ** argv) { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = home + "/.nix-channels"; - nixDefExpr = home + "/.nix-defexpr"; + channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; + nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; enum { cNone, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 776c5f6db..0daf374de 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -478,9 +478,14 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector<DerivedPath> targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{*drvPath}); + targets.push_back(DerivedPath::Built{ + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); else - targets.push_back(DerivedPath::Opaque{i.queryOutPath()}); + targets.push_back(DerivedPath::Opaque{ + .path = i.queryOutPath(), + }); printMissing(state.store, targets); } @@ -647,7 +652,7 @@ static void upgradeDerivations(Globals & globals, } else newElems.push_back(i); } catch (Error & e) { - e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName()); + e.addTrace(nullptr, "while trying to find an upgrade for '%s'", i.queryName()); throw; } } @@ -751,8 +756,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) auto drvPath = drv.queryDrvPath(); std::vector<DerivedPath> paths { drvPath - ? (DerivedPath) (DerivedPath::Built { *drvPath }) - : (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }), + ? (DerivedPath) (DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }) + : (DerivedPath) (DerivedPath::Opaque { + .path = drv.queryOutPath(), + }), }; printMissing(globals.state->store, paths); if (globals.dryRun) return; @@ -958,7 +968,7 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); } catch (Error & e) { - e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName()); + e.addTrace(nullptr, "while querying the derivation named '%1%'", i.queryName()); throw; } } @@ -1262,7 +1272,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); } catch (Error & e) { - e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName()); + e.addTrace(nullptr, "while querying the derivation named '%1%'", i.queryName()); throw; } } @@ -1279,7 +1289,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1383,7 +1393,10 @@ static int main_nix_env(int argc, char * * argv) Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; + { + Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + globals.instSource.nixExprPath = nixExprPath; + } globals.instSource.systemFilter = "*"; if (!pathExists(globals.instSource.nixExprPath)) { diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 4b1202be3..cad7f9c88 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -134,9 +134,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.forceValue(topLevel, [&]() { return topLevel.determinePos(noPos); }); PathSet context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context); + auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); - auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context); + auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, ""); /* Realise the resulting store expression. */ debug("building user environment"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b59a6d026..3bbefedbe 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -808,14 +808,23 @@ static void opServe(Strings opFlags, Strings opArgs) if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.maxLogSize = readNum<unsigned long>(in); if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - settings.buildRepeat = readInt(in); - settings.enforceDeterminism = readInt(in); + auto nrRepeats = readInt(in); + if (nrRepeats != 0) { + throw Error("client requested repeating builds, but this is not currently implemented"); + } + // Ignore 'enforceDeterminism'. It used to be true by + // default, but also only never had any effect when + // `nrRepeats == 0`. We have already asserted that + // `nrRepeats` in fact is 0, so we can safely ignore this + // without doing something other than what the client + // asked for. + readInt(in); + settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { settings.keepFailed = (bool) readInt(in); } - settings.printRepeatedBuilds = false; }; while (true) { @@ -926,7 +935,6 @@ static void opServe(Strings opFlags, Strings opArgs) worker_proto::write(*store, out, status.builtOutputs); } - break; } diff --git a/src/nix/app.cc b/src/nix/app.cc index 5658f2a52..5cd65136f 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,4 +1,5 @@ #include "installables.hh" +#include "installable-derived-path.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" @@ -8,31 +9,6 @@ namespace nix { -struct InstallableDerivedPath : Installable -{ - ref<Store> store; - const DerivedPath derivedPath; - - InstallableDerivedPath(ref<Store> store, const DerivedPath & derivedPath) - : store(store) - , derivedPath(derivedPath) - { - } - - - std::string what() const override { return derivedPath.to_string(*store); } - - DerivedPaths toDerivedPaths() override - { - return {derivedPath}; - } - - std::optional<StorePath> getStorePath() override - { - return std::nullopt; - } -}; - /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. @@ -80,9 +56,29 @@ UnresolvedApp Installable::toApp(EvalState & state) if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector<StorePathWithOutputs> context2; - for (auto & [path, name] : context) - context2.push_back({path, {name}}); + std::vector<DerivedPath> context2; + for (auto & c : context) { + context2.emplace_back(std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { + /* We want all outputs of the drv */ + return DerivedPath::Built { + .drvPath = d.drvPath, + .outputs = OutputsSpec::All {}, + }; + }, + [&](const NixStringContextElem::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + [&](const NixStringContextElem::Opaque & o) -> DerivedPath { + return DerivedPath::Opaque { + .path = o.path, + }; + }, + }, c.raw())); + } return UnresolvedApp{App { .context = std::move(context2), @@ -106,7 +102,10 @@ UnresolvedApp Installable::toApp(EvalState & state) : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { - .context = { { drvPath, {outputName} } }, + .context = { DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName }, + } }, .program = program, }}; } @@ -124,7 +123,7 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store) for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared<InstallableDerivedPath>(store, ctxElt.toDerivedPath())); + std::make_shared<InstallableDerivedPath>(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff --git a/src/nix/build.cc b/src/nix/build.cc index 94b169167..12b22d999 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (dryRun) { std::vector<DerivedPath> pathsToBuild; - for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); - } + for (auto & i : installables) + for (auto & b : i->toDerivedPaths()) + pathsToBuild.push_back(b.path); + printMissing(store, pathsToBuild, lvlError); + if (json) logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2e48e4c74..dcf9a6f2d 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -75,10 +76,10 @@ struct CmdBundle : InstallableCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec, + evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec, {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, @@ -97,15 +98,20 @@ struct CmdBundle : InstallableCommand throw Error("the bundler '%s' does not produce a derivation", bundler.what()); PathSet context2; - auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2); + auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); auto attr2 = vRes->attrs->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2); + auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); - store->buildPaths({ DerivedPath::Built { drvPath } }); + store->buildPaths({ + DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }); auto outPathS = store->printStorePath(outPath); @@ -113,7 +119,7 @@ struct CmdBundle : InstallableCommand auto * attr = vRes->attrs->get(evalState->sName); if (!attr) throw Error("attribute 'name' missing"); - outLink = evalState->forceStringNoCtx(*attr->value, attr->pos); + outLink = evalState->forceStringNoCtx(*attr->value, attr->pos, ""); } // TODO: will crash if not a localFSStore? diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 940923d3b..a22bccba1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -34,6 +34,43 @@ using namespace nix; using namespace nix::daemon; +struct AuthorizationSettings : Config { + + Setting<Strings> trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting<Strings> allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; +}; + +AuthorizationSettings authorizationSettings; + +static GlobalConfig::Register rSettings(&authorizationSettings); + #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) @@ -203,8 +240,8 @@ static void daemonLoop() struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; + Strings trustedUsers = authorizationSettings.trustedUsers; + Strings allowedUsers = authorizationSettings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = Trusted; @@ -241,15 +278,7 @@ static void daemonLoop() // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - store.createUser(user, peer.uid); - }); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); }, options); @@ -257,7 +286,7 @@ static void daemonLoop() } catch (Interrupted & e) { return; } catch (Error & error) { - ErrorInfo ei = error.info(); + auto ei = error.info(); // FIXME: add to trace? ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); logError(ei); @@ -302,7 +331,7 @@ static void runDaemon(bool stdio) /* Auth hook is empty because in this mode we blindly trust the standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); } } else daemonLoop(); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 4de109754..9d07a7a85 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,9 +1,10 @@ #include "eval.hh" #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" @@ -164,6 +165,14 @@ struct BuildEnvironment { return vars == other.vars && bashFunctions == other.bashFunctions; } + + std::string getSystem() const + { + if (auto v = get(vars, "system")) + return getString(*v); + else + return settings.thisSystem; + } }; const static std::string getEnvSh = @@ -192,10 +201,12 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore drv.env.erase("allowedRequisites"); drv.env.erase("disallowedReferences"); drv.env.erase("disallowedRequisites"); + drv.env.erase("name"); /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; + drv.env.emplace("name", drv.name); drv.inputSrcs.insert(std::move(getEnvShPath)); if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { @@ -222,7 +233,12 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); + store->buildPaths( + { DerivedPath::Built { + .drvPath = shellDrvPath, + .outputs = OutputsSpec::All { }, + }}, + bmNormal, evalStore); for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); @@ -568,7 +584,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, shell, args); + runProgramInStore(store, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0621d662c..3489cc132 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -106,7 +106,7 @@ void printClosureDiff( using namespace nix; -struct CmdDiffClosures : SourceExprCommand +struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 76a134b1f..dfe75fbdf 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "editor-for.hh" #include <unistd.h> diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ba82b5772..a579213fd 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -11,13 +11,13 @@ using namespace nix; -struct CmdEval : MixJSON, InstallableCommand +struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption { bool raw = false; std::optional<std::string> apply; std::optional<Path> writeTo; - CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) + CmdEval() : InstallableCommand() { addFlag({ .longName = "raw", @@ -112,7 +112,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (raw) { stopProgressBar(); - std::cout << *state->coerceToString(noPos, *v, context); + std::cout << *state->coerceToString(noPos, *v, context, "while generating the eval command output"); } else if (json) { diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 2ee8a707d..8c6042d94 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -16,7 +16,7 @@ R""( # Description This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every mutable input (like `nixpkgs`) to its +updating the lock for every unlocked input (like `nixpkgs`) to its current version. This is equivalent to passing `--recreate-lock-file` to any command that operates on a flake. That is, diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 336f6723a..053a9c9e1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "eval.hh" @@ -7,7 +8,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" @@ -126,12 +127,12 @@ static void enumerateOutputs(EvalState & state, Value & vFlake, std::function<void(const std::string & name, Value & vProvide, const PosIdx pos)> callback) { auto pos = vFlake.determinePos(noPos); - state.forceAttrs(vFlake, pos); + state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); assert(aOutputs); - state.forceAttrs(*aOutputs->value, pos); + state.forceAttrs(*aOutputs->value, pos, "while evaluating the outputs of a flake"); auto sHydraJobs = state.symbols.create("hydraJobs"); @@ -215,7 +216,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); - std::unordered_set<std::shared_ptr<Node>> visited; + std::set<ref<Node>> visited; std::function<void(const Node & node, const std::string & prefix)> recurse; @@ -227,7 +228,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto lockedNode = std::get_if<0>(&input.second)) { logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, - *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); + (*lockedNode)->lockedRef); bool firstVisit = visited.insert(*lockedNode).second; @@ -348,7 +349,7 @@ struct CmdFlakeCheck : FlakeCommand // FIXME auto app = App(*state, v); for (auto & i : app.context) { - auto [drvPathS, outputName] = decodeContext(i); + auto [drvPathS, outputName] = NixStringContextElem::parse(i); store->parseStorePath(drvPathS); } #endif @@ -381,23 +382,6 @@ struct CmdFlakeCheck : FlakeCommand auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); - if (v.isLambda()) { - if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis) - throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.type() == nAttrs) { - for (auto & attr : *v.attrs) - try { - state->forceValue(*attr.value, attr.pos); - } catch (Error & e) { - e.addTrace( - state->positions[attr.pos], - hintfmt("while evaluating the option '%s'", state->symbols[attr.name])); - throw; - } - } else - throw Error("module must be a function or an attribute set"); - // FIXME: if we have a 'nixpkgs' input, use it to - // check the module. } catch (Error & e) { e.addTrace(resolve(pos), hintfmt("while checking the NixOS module '%s'", attrPath)); reportError(e); @@ -408,13 +392,13 @@ struct CmdFlakeCheck : FlakeCommand checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - state->forceAttrs(v, pos); + state->forceAttrs(v, pos, ""); if (state->isDerivation(v)) throw Error("jobset should not be a derivation at top-level"); for (auto & attr : *v.attrs) { - state->forceAttrs(*attr.value, attr.pos); + state->forceAttrs(*attr.value, attr.pos, ""); auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); if (state->isDerivation(*attr.value)) { Activity act(*logger, lvlChatty, actUnknown, @@ -436,7 +420,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("checking NixOS configuration '%s'", attrPath)); Bindings & bindings(*state->allocBindings(0)); auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; - state->forceAttrs(*vToplevel, pos); + state->forceValue(*vToplevel, pos); if (!state->isDerivation(*vToplevel)) throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { @@ -450,12 +434,12 @@ struct CmdFlakeCheck : FlakeCommand Activity act(*logger, lvlChatty, actUnknown, fmt("checking template '%s'", attrPath)); - state->forceAttrs(v, pos); + state->forceAttrs(v, pos, ""); if (auto attr = v.attrs->get(state->symbols.create("path"))) { if (attr->name == state->symbols.create("path")) { PathSet context; - auto path = state->coerceToPath(attr->pos, *attr->value, context); + auto path = state->coerceToPath(attr->pos, *attr->value, context, ""); if (!store->isInStore(path)) throw Error("template '%s' has a bad 'path' attribute"); // TODO: recursively check the flake in 'path'. @@ -464,7 +448,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("template '%s' lacks attribute 'path'", attrPath); if (auto attr = v.attrs->get(state->symbols.create("description"))) - state->forceStringNoCtx(*attr->value, attr->pos); + state->forceStringNoCtx(*attr->value, attr->pos, ""); else throw Error("template '%s' lacks attribute 'description'", attrPath); @@ -521,23 +505,27 @@ struct CmdFlakeCheck : FlakeCommand warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); if (name == "checks") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); + state->forceAttrs(*attr.value, attr.pos, ""); for (auto & attr2 : *attr.value->attrs) { auto drvPath = checkDerivation( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) - drvPaths.push_back(DerivedPath::Built{*drvPath}); + if (drvPath && attr_name == settings.thisSystem.get()) { + drvPaths.push_back(DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); + } } } } else if (name == "formatter") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); @@ -548,11 +536,11 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "packages" || name == "devShells") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); + state->forceAttrs(*attr.value, attr.pos, ""); for (auto & attr2 : *attr.value->attrs) checkDerivation( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), @@ -561,11 +549,11 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "apps") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); + state->forceAttrs(*attr.value, attr.pos, ""); for (auto & attr2 : *attr.value->attrs) checkApp( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), @@ -574,7 +562,7 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "defaultPackage" || name == "devShell") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); @@ -585,7 +573,7 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "defaultApp") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); @@ -596,7 +584,7 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "legacyPackages") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { checkSystemName(state->symbols[attr.name], attr.pos); // FIXME: do getDerivations? @@ -607,7 +595,7 @@ struct CmdFlakeCheck : FlakeCommand checkOverlay(name, vOutput, pos); else if (name == "overlays") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); @@ -617,14 +605,14 @@ struct CmdFlakeCheck : FlakeCommand checkModule(name, vOutput, pos); else if (name == "nixosModules") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) checkModule(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } else if (name == "nixosConfigurations") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) checkNixOSConfiguration(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); @@ -637,14 +625,14 @@ struct CmdFlakeCheck : FlakeCommand checkTemplate(name, vOutput, pos); else if (name == "templates") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); } else if (name == "defaultBundler") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); @@ -655,11 +643,11 @@ struct CmdFlakeCheck : FlakeCommand } else if (name == "bundlers") { - state->forceAttrs(vOutput, pos); + state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos); + state->forceAttrs(*attr.value, attr.pos, ""); for (auto & attr2 : *attr.value->attrs) { checkBundler( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), @@ -668,6 +656,19 @@ struct CmdFlakeCheck : FlakeCommand } } + else if ( + name == "lib" + || name == "darwinConfigurations" + || name == "darwinModules" + || name == "flakeModule" + || name == "flakeModules" + || name == "herculesCI" + || name == "homeConfigurations" + || name == "nixopsConfigurations" + ) + // Known but unchecked community attribute + ; + else warn("unknown flake output '%s'", name); @@ -966,6 +967,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -974,6 +976,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} + }); } std::string description() override @@ -994,6 +1001,62 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = std::make_shared<LockedFlake>(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); + + std::function<bool( + eval_cache::AttrCursor & visitor, + const std::vector<Symbol> &attrPath, + const Symbol &attr)> hasContent; + + // For frameworks it's important that structures are as lazy as possible + // to prevent infinite recursions, performance issues and errors that + // aren't related to the thing to evaluate. As a consequence, they have + // to emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not useful to the user + // so we omit them. + hasContent = [&]( + eval_cache::AttrCursor & visitor, + const std::vector<Symbol> &attrPath, + const Symbol &attr) -> bool + { + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto attrPathS = state->symbols.resolve(attrPath2); + const auto & attrName = state->symbols[attr]; + + auto visitor2 = visitor.getAttr(attrName); + + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + // If we don't recognize it, it's probably content + return true; + }; std::function<nlohmann::json( eval_cache::AttrCursor & visitor, @@ -1020,7 +1083,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { if (!json) logger->cout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + std::vector<Symbol> attrs; + for (const auto &attr : visitor.getAttrs()) { + if (hasContent(visitor, attrPath, attr)) + attrs.push_back(attr); + } + for (const auto & [i, attr] : enumerate(attrs)) { const auto & attrName = state->symbols[attr]; bool last = i + 1 == attrs.size(); @@ -1084,10 +1152,18 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (visitor.isDerivation()) - showDerivation(); - else - throw Error("expected a derivation"); + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { @@ -1106,6 +1182,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else { logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } } else { if (visitor.isDerivation()) showDerivation(); diff --git a/src/nix/flake.md b/src/nix/flake.md index a1ab43281..810e9ebea 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -18,51 +18,56 @@ values such as packages or NixOS modules provided by the flake). Flake references (*flakerefs*) are a way to specify the location of a flake. These have two different forms: -* An attribute set representation, e.g. - ```nix - { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - } - ``` +## Attribute set representation - The only required attribute is `type`. The supported types are - listed below. +Example: -* A URL-like syntax, e.g. +```nix +{ + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; +} +``` - ``` - github:NixOS/nixpkgs - ``` +The only required attribute is `type`. The supported types are +listed below. - These are used on the command line as a more convenient alternative - to the attribute set representation. For instance, in the command +## URL-like syntax - ```console - # nix build github:NixOS/nixpkgs#hello - ``` +Example: - `github:NixOS/nixpkgs` is a flake reference (while `hello` is an - output attribute). They are also allowed in the `inputs` attribute - of a flake, e.g. +``` +github:NixOS/nixpkgs +``` - ```nix - inputs.nixpkgs.url = github:NixOS/nixpkgs; - ``` +These are used on the command line as a more convenient alternative +to the attribute set representation. For instance, in the command - is equivalent to +```console +# nix build github:NixOS/nixpkgs#hello +``` - ```nix - inputs.nixpkgs = { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - }; - ``` +`github:NixOS/nixpkgs` is a flake reference (while `hello` is an +output attribute). They are also allowed in the `inputs` attribute +of a flake, e.g. + +```nix +inputs.nixpkgs.url = github:NixOS/nixpkgs; +``` + +is equivalent to + +```nix +inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; +}; +``` -## Examples +### Examples Here are some examples of flake references in their URL-like representation: diff --git a/src/nix/log.cc b/src/nix/log.cc index 72d02ef11..a0598ca13 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand [&](const DerivedPath::Built & bfd) { return logSub.getBuildLog(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 2c6309c81..d3d2f5b16 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -199,7 +199,7 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve if (!attr) throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); - auto markdown = state.forceString(*attr->value); + auto markdown = state.forceString(*attr->value, noPos, "while evaluating the lowdown help text"); RunPager pager; std::cout << renderMarkdownToTerminal(markdown) << "\n"; diff --git a/src/nix/nix.md b/src/nix/nix.md index d48682a94..db60c59ff 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -115,12 +115,11 @@ the Nix store. Here are the recognised types of installables: * **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` - Store derivations are store paths with extension `.drv` and are a - low-level representation of a build-time dependency graph used - internally by Nix. By default, if you pass a store derivation to a - `nix` subcommand, it will operate on the *output paths* of the - derivation. For example, `nix path-info` prints information about - the output paths: + By default, if you pass a [store derivation] path to a `nix` subcommand, the command will operate on the [output path]s of the derivation. + + [output path]: ../../glossary.md#gloss-output-path + + For example, `nix path-info` prints information about the output paths: ```console # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv @@ -164,6 +163,13 @@ operate are determined as follows: … ``` + and likewise, using a store path to a "drv" file to specify the derivation: + + ```console + # nix build '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^dev,static' + … + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: @@ -177,6 +183,12 @@ operate are determined as follows: /nix/store/q6580lr01jpcsqs4r5arlh4ki2c1m9rv-glibc-2.33-123-dev 44200560 ``` + and likewise, using a store path to a "drv" file to specify the derivation: + + ```console + # nix path-info -S '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*' + … + ``` * If you didn't specify the desired outputs, but the derivation has an attribute `meta.outputsToInstall`, Nix will use those outputs. For example, since the package `nixpkgs#libxml2` has this attribute: @@ -189,6 +201,11 @@ operate are determined as follows: a command like `nix shell nixpkgs#libxml2` will provide only those two outputs by default. + Note that a [store derivation] (given by its `.drv` file store path) doesn't have + any attributes like `meta`, and thus this case doesn't apply to it. + + [store derivation]: ../../glossary.md#gloss-store-derivation + * Otherwise, Nix will use all outputs of the derivation. # Nix stores diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 7a1714ba4..b30898ac0 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -68,7 +68,9 @@ R""( ] ``` -* Print the path of the store derivation produced by `nixpkgs#hello`: +* Print the path of the [store derivation] produced by `nixpkgs#hello`: + + [store derivation]: ../../glossary.md#gloss-store-derivation ```console # nix path-info --derivation nixpkgs#hello diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3c3b7bb45..5c44510ab 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -1,10 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "finally.hh" + +#include <nlohmann/json.hpp> using namespace nix; -struct CmdPingStore : StoreCommand +struct CmdPingStore : StoreCommand, MixJSON { std::string description() override { @@ -20,10 +23,21 @@ struct CmdPingStore : StoreCommand void run(ref<Store> store) override { - notice("Store URL: %s", store->getUri()); - store->connect(); - if (auto version = store->getVersion()) - notice("Version: %s", *version); + if (!json) { + notice("Store URL: %s", store->getUri()); + store->connect(); + if (auto version = store->getVersion()) + notice("Version: %s", *version); + } else { + nlohmann::json res; + Finally printRes([&]() { + logger->cout("%s", res); + }); + res["url"] = store->getUri(); + store->connect(); + if (auto version = store->getVersion()) + res["version"] = *version; + } } }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce3288dc1..fc3823406 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -28,17 +28,17 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) Value vMirrors; // FIXME: use nixpkgs flake state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors); - state.forceAttrs(vMirrors, noPos); + state.forceAttrs(vMirrors, noPos, "while evaluating the set of all mirrors"); auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); if (mirrorList == vMirrors.attrs->end()) throw Error("unknown mirror name '%s'", mirrorName); - state.forceList(*mirrorList->value, noPos); + state.forceList(*mirrorList->value, noPos, "while evaluating one mirror configuration"); if (mirrorList->value->listSize() < 1) throw Error("mirror URL '%s' did not expand to anything", url); - std::string mirror(state.forceString(*mirrorList->value->listElems()[0])); + std::string mirror(state.forceString(*mirrorList->value->listElems()[0], noPos, "while evaluating the first available mirror")); return mirror + (hasSuffix(mirror, "/") ? "" : "/") + s.substr(p + 1); } @@ -196,29 +196,29 @@ static int main_nix_prefetch_url(int argc, char * * argv) Value vRoot; state->evalFile(path, vRoot); Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v, noPos); + state->forceAttrs(v, noPos, "while evaluating the source attribute to prefetch"); /* Extract the URL. */ auto * attr = v.attrs->get(state->symbols.create("urls")); if (!attr) throw Error("attribute 'urls' missing"); - state->forceList(*attr->value, noPos); + state->forceList(*attr->value, noPos, "while evaluating the urls to prefetch"); if (attr->value->listSize() < 1) throw Error("'urls' list is empty"); - url = state->forceString(*attr->value->listElems()[0]); + url = state->forceString(*attr->value->listElems()[0], noPos, "while evaluating the first url from the urls list"); /* Extract the hash mode. */ auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); if (!attr2) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state->forceString(*attr2->value) == "recursive"; + unpack = state->forceString(*attr2->value, noPos, "while evaluating the outputHashMode of the source to prefetch") == "recursive"; /* Extract the name. */ if (!name) { auto attr3 = v.attrs->get(state->symbols.create("name")); if (!attr3) - name = state->forceString(*attr3->value); + name = state->forceString(*attr3->value, noPos, "while evaluating the name of the source to prefetch"); } } diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index bdab9a208..fa786162f 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -20,11 +20,11 @@ following fields: * An integer that can be used to unambiguously identify the package in invocations of `nix profile remove` and `nix profile upgrade`. -* The original ("mutable") flake reference and output attribute path +* The original ("unlocked") flake reference and output attribute path used at installation time. -* The immutable flake reference to which the mutable flake reference - was resolved. +* The locked flake reference to which the unlocked flake reference was + resolved. * The store path(s) of the package. diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index e06e74abe..39cca428b 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -2,7 +2,7 @@ R""( # Examples -* Upgrade all packages that were installed using a mutable flake +* Upgrade all packages that were installed using an unlocked flake reference: ```console @@ -32,9 +32,9 @@ the package was installed. > **Warning** > -> This only works if you used a *mutable* flake reference at +> This only works if you used an *unlocked* flake reference at > installation time, e.g. `nixpkgs#hello`. It does not work if you -> used an *immutable* flake reference +> used a *locked* flake reference > (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), > since in that case the "latest version" is always the same. diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 11910523d..208542a5c 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -22,7 +23,7 @@ struct ProfileElementSource // FIXME: record original attrpath. FlakeRef resolvedRef; std::string attrPath; - OutputsSpec outputs; + ExtendedOutputsSpec outputs; bool operator < (const ProfileElementSource & other) const { @@ -32,17 +33,19 @@ struct ProfileElementSource } }; +const int defaultPriority = 5; + struct ProfileElement { StorePathSet storePaths; std::optional<ProfileElementSource> source; bool active = true; - int priority = 5; + int priority = defaultPriority; std::string describe() const { if (source) - return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs)); + return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string()); StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); @@ -124,7 +127,7 @@ struct ProfileManifest parseFlakeRef(e[sOriginalUrl]), parseFlakeRef(e[sUrl]), e["attrPath"], - e["outputs"].get<OutputsSpec>() + e["outputs"].get<ExtendedOutputsSpec>() }; } elements.emplace_back(std::move(element)); @@ -251,13 +254,20 @@ struct ProfileManifest } }; -static std::map<Installable *, BuiltPaths> +static std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> builtPathsPerInstallable( const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths) { - std::map<Installable *, BuiltPaths> res; - for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath.path); + std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> res; + for (auto & [installable, builtPath] : builtPaths) { + auto & r = res[installable.get()]; + /* Note that there could be conflicting info + (e.g. meta.priority fields) if the installable returned + multiple derivations. So pick one arbitrarily. FIXME: + print a warning? */ + r.first.push_back(builtPath.path); + r.second = builtPath.info; + } return res; } @@ -297,28 +307,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + auto & [res, info] = builtPaths[installable.get()]; - - if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { - // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + if (info.originalRef && info.resolvedRef && info.attrPath && info.extendedOutputsSpec) { element.source = ProfileElementSource { - installable2->flakeRef, - resolvedRef, - attrPath, - installable2->outputsSpec + .originalRef = *info.originalRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = *info.extendedOutputsSpec, }; - - if(drv.priority) { - element.priority = *drv.priority; - } } - if(priority) { // if --priority was specified we want to override the priority of the installable - element.priority = *priority; - }; + // If --priority was specified we want to override the + // priority of the installable. + element.priority = + priority + ? *priority + : info.priority.value_or(defaultPriority); - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, res); manifest.elements.push_back(std::move(element)); } @@ -476,18 +483,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto & info = derivedPaths[0].info; + + assert(info.resolvedRef && info.attrPath); - if (element.source->resolvedRef == resolvedRef) continue; + if (element.source->resolvedRef == info.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, resolvedRef); + element.source->attrPath, element.source->resolvedRef, *info.resolvedRef); element.source = ProfileElementSource { - installable->flakeRef, - resolvedRef, - attrPath, - installable->outputsSpec + .originalRef = installable->flakeRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = installable->extendedOutputsSpec, }; installables.push_back(installable); @@ -515,7 +526,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first); } updateProfile(manifest.build(store)); @@ -543,8 +554,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } diff --git a/src/nix/profile.md b/src/nix/profile.md index be3c5ba1a..273e02280 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -88,8 +88,7 @@ has the following fields: the user at the time of installation (e.g. `nixpkgs`). This is also the flake reference that will be used by `nix profile upgrade`. -* `uri`: The immutable flake reference to which `originalUrl` - resolved. +* `uri`: The locked flake reference to which `originalUrl` resolved. * `attrPath`: The flake output attribute that provided this package. Note that this is not necessarily the attribute that the diff --git a/src/nix/registry.cc b/src/nix/registry.cc index c496f94f8..b5bdfba95 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -183,14 +183,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand void run(nix::ref<nix::Store> store) override { - if (locked.empty()) { - locked = url; - } + if (locked.empty()) locked = url; auto registry = getRegistry(); auto ref = parseFlakeRef(url); - auto locked_ref = parseFlakeRef(locked); + auto lockedRef = parseFlakeRef(locked); registry->remove(ref.input); - auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); + auto [tree, resolved] = lockedRef.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; registry->add(ref.input, resolved, extraAttrs); diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 000000000..679bdea77 --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,95 @@ +#include "eval.hh" +#include "globals.hh" +#include "command.hh" +#include "repl.hh" + +namespace nix { + +struct CmdRepl : InstallablesCommand +{ + CmdRepl() { + evalSettings.pureEval = false; + } + + void prepare() override + { + if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { + warn("future versions of Nix will require using `--file` to load a file"); + if (this->_installables.size() > 1) + warn("more than one input file is not currently supported"); + auto filePath = this->_installables[0].data(); + file = std::optional(filePath); + _installables.front() = _installables.back(); + _installables.pop_back(); + } + installables = InstallablesCommand::load(); + } + + std::vector<std::string> files; + + Strings getDefaultFlakeAttrPaths() override + { + return {""}; + } + + bool useDefaultInstallables() override + { + return file.has_value() or expr.has_value(); + } + + bool forceImpureByDefault() override + { + return true; + } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + std::string doc() override + { + return + #include "repl.md" + ; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + auto getValues = [&]()->AbstractNixRepl::AnnotatedValues{ + auto installables = load(); + AbstractNixRepl::AnnotatedValues values; + for (auto & installable: installables){ + auto what = installable->what(); + if (file){ + auto [val, pos] = installable->toValue(*state); + auto what = installable->what(); + state->forceValue(*val, pos); + auto autoArgs = getAutoArgs(*state); + auto valPost = state->allocValue(); + state->autoCallFunction(*autoArgs, *val, *valPost); + state->forceValue(*valPost, pos); + values.push_back( {valPost, what }); + } else { + auto [val, pos] = installable->toValue(*state); + values.push_back( {val, what} ); + } + } + return values; + }; + auto repl = AbstractNixRepl::create( + searchPath, + openStore(), + state, + getValues + ); + repl->autoArgs = getAutoArgs(*repl->state); + repl->initEnv(); + repl->mainLoop(); + } +}; + +static auto rCmdRepl = registerCommand<CmdRepl>("repl"); + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index 45d2dfd0d..6fca68047 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -9,6 +9,7 @@ #include "fs-accessor.hh" #include "progress-bar.hh" #include "eval.hh" +#include "build/personality.hh" #if __linux__ #include <sys/mount.h> @@ -24,7 +25,8 @@ namespace nix { void runProgramInStore(ref<Store> store, const std::string & program, - const Strings & args) + const Strings & args, + std::optional<std::string_view> system) { stopProgressBar(); @@ -44,7 +46,7 @@ void runProgramInStore(ref<Store> store, throw Error("store '%s' is not a local store so it does not support command execution", store->getUri()); if (store->storeDir != store2->getRealStoreDir()) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), std::string(system.value_or("")), program }; for (auto & arg : args) helperArgs.push_back(arg); execv(getSelfExe().value_or("nix").c_str(), stringsToCharPtrs(helperArgs).data()); @@ -52,6 +54,9 @@ void runProgramInStore(ref<Store> store, throw SysError("could not execute chroot helper"); } + if (system) + setPersonality(*system); + execvp(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); @@ -199,6 +204,7 @@ void chrootHelper(int argc, char * * argv) int p = 1; std::string storeDir = argv[p++]; std::string realStoreDir = argv[p++]; + std::string system = argv[p++]; std::string cmd = argv[p++]; Strings args; while (p < argc) @@ -262,6 +268,9 @@ void chrootHelper(int argc, char * * argv) writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + if (system != "") + setPersonality(system); + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to exec '%s'", cmd); diff --git a/src/nix/run.hh b/src/nix/run.hh index 6180a87dd..fed360158 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -6,6 +6,7 @@ namespace nix { void runProgramInStore(ref<Store> store, const std::string & program, - const Strings & args); + const Strings & args, + std::optional<std::string_view> system = std::nullopt); } diff --git a/src/nix/search.cc b/src/nix/search.cc index d2a31607d..4fa1e7837 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -56,8 +56,8 @@ struct CmdSearch : InstallableCommand, MixJSON Strings getDefaultFlakeAttrPaths() override { return { - "packages." + settings.thisSystem.get() + ".", - "legacyPackages." + settings.thisSystem.get() + "." + "packages." + settings.thisSystem.get(), + "legacyPackages." + settings.thisSystem.get() }; } diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 29944e748..3530584f9 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -9,15 +9,44 @@ using namespace nix; struct CmdShowConfig : Command, MixJSON { + std::optional<std::string> name; + + CmdShowConfig() { + expectArgs({ + .label = {"name"}, + .optional = true, + .handler = {&name}, + }); + } + std::string description() override { - return "show the Nix configuration"; + return "show the Nix configuration or the value of a specific setting"; } Category category() override { return catUtility; } void run() override { + if (name) { + if (json) { + throw UsageError("'--json' is not supported when specifying a setting name"); + } + + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + auto setting = settings.find(*name); + + if (setting == settings.end()) { + throw Error("could not find setting '%1%'", *name); + } else { + const auto & value = setting->second.value; + logger->cout("%s", value); + } + + return; + } + if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). logger->cout("%s", globalConfig.toJSON().dump()); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index af2e676a4..d1a516cad 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -54,56 +54,8 @@ struct CmdShowDerivation : InstallablesCommand for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - json& drvObj = jsonRoot[store->printStorePath(drvPath)]; - - auto drv = store->readDerivation(drvPath); - - { - json& outputsObj = drvObj["outputs"]; - outputsObj = json::object(); - for (auto & [_outputName, output] : drv.outputs) { - auto & outputName = _outputName; // work around clang bug - auto& outputObj = outputsObj[outputName]; - outputObj = json::object(); - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doi) { - outputObj["path"] = store->printStorePath(doi.path); - }, - [&](const DerivationOutput::CAFixed & dof) { - outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName)); - outputObj["hashAlgo"] = dof.hash.printMethodAlgo(); - outputObj["hash"] = dof.hash.hash.to_string(Base16, false); - }, - [&](const DerivationOutput::CAFloating & dof) { - outputObj["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); - }, - [&](const DerivationOutput::Deferred &) {}, - [&](const DerivationOutput::Impure & doi) { - outputObj["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); - outputObj["impure"] = true; - }, - }, output.raw()); - } - } - - { - auto& inputsList = drvObj["inputSrcs"]; - inputsList = json::array(); - for (auto & input : drv.inputSrcs) - inputsList.emplace_back(store->printStorePath(input)); - } - - { - auto& inputDrvsObj = drvObj["inputDrvs"]; - inputDrvsObj = json::object(); - for (auto & input : drv.inputDrvs) - inputDrvsObj[store->printStorePath(input.first)] = input.second; - } - - drvObj["system"] = drv.platform; - drvObj["builder"] = drv.builder; - drvObj["args"] = drv.args; - drvObj["env"] = drv.env; + jsonRoot[store->printStorePath(drvPath)] = + store->readDerivation(drvPath).toJSON(*store); } std::cout << jsonRoot.dump(2) << std::endl; } diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md index aa863899c..2cd93aa62 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/show-derivation.md @@ -2,9 +2,11 @@ R""( # Examples -* Show the store derivation that results from evaluating the Hello +* Show the [store derivation] that results from evaluating the Hello package: + [store derivation]: ../../glossary.md#gloss-store-derivation + ```console # nix show-derivation nixpkgs#hello { @@ -37,7 +39,7 @@ R""( # Description This command prints on standard output a JSON representation of the -store derivations to which *installables* evaluate. Store derivations +[store derivation]s to which *installables* evaluate. Store derivations are used internally by Nix. They are store paths with extension `.drv` that represent the build-time dependency graph to which a Nix expression evaluates. diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 2e288f743..d5fab5f2f 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand auto dstStore = getDstStore(); auto & dstLogStore = require<LogStore>(*dstStore); - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & drvPath : i->toDrvPaths(getEvalStore())) - drvPaths.insert(drvPath); - - for (auto & drvPath : drvPaths) { + for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) { if (auto log = srcLogStore.getBuildLog(drvPath)) dstLogStore.addBuildLog(drvPath, *log); else diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md index 19ae57079..0937250f2 100644 --- a/src/nix/store-copy-log.md +++ b/src/nix/store-copy-log.md @@ -18,7 +18,9 @@ R""( (The flag `--substituters ''` avoids querying `https://cache.nixos.org` for the log.) -* To copy the log for a specific store derivation via SSH: +* To copy the log for a specific [store derivation] via SSH: + + [store derivation]: ../../glossary.md#gloss-store-derivation ```console # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 2d2453395..17796d6b8 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -144,7 +144,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Bindings & bindings(*state->allocBindings(0)); auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; - return store->parseStorePath(state->forceString(*v2)); + return store->parseStorePath(state->forceString(*v2, noPos, "while evaluating the path tho latest nix version")); } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index efa2434dc..0b306cc11 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -81,14 +81,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount<std::atomic<size_t>> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 723017497..a3a9dc698 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -27,7 +27,7 @@ static std::string filterPrintable(const std::string & s) return res; } -struct CmdWhyDepends : SourceExprCommand +struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions { std::string _package, _dependency; bool all = false; @@ -95,23 +95,13 @@ struct CmdWhyDepends : SourceExprCommand * to build. */ auto dependency = parseInstallable(store, _dependency); - auto derivedDependency = dependency->toDerivedPath(); - auto optDependencyPath = std::visit(overloaded { - [](const DerivedPath::Opaque & nodrv) -> std::optional<StorePath> { - return { nodrv.path }; - }, - [&](const DerivedPath::Built & hasdrv) -> std::optional<StorePath> { - if (hasdrv.outputs.size() != 1) { - throw Error("argument '%s' should evaluate to one store path", dependency->what()); - } - auto outputMap = store->queryPartialDerivationOutputMap(hasdrv.drvPath); - auto maybePath = outputMap.find(*hasdrv.outputs.begin()); - if (maybePath == outputMap.end()) { - throw Error("unexpected end of iterator"); - } - return maybePath->second; - }, - }, derivedDependency.raw()); + auto optDependencyPath = [&]() -> std::optional<StorePath> { + try { + return {Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency)}; + } catch (MissingRealisation &) { + return std::nullopt; + } + }(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); |