diff options
Diffstat (limited to 'src')
287 files changed, 4959 insertions, 2489 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 6b81ecc49..cfc4baaca 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -72,6 +72,7 @@ static int main_build_remote(int argc, char * * argv) settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -112,10 +113,14 @@ static int main_build_remote(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -214,7 +219,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep<StringSet>(", ", m.supportedFeatures) % concatStringsSep<StringSet>(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error.str()); std::cerr << "# decline\n"; } @@ -300,7 +305,7 @@ connected: std::set<Realisation> missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; @@ -332,7 +337,7 @@ connected: for (auto & realisation : missingRealisations) { // Should hold, because if the feature isn't enabled the set // of missing realisations should be empty - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); store->registerDrvOutput(realisation); } diff --git a/src/libcmd/command-installable-value.cc b/src/libcmd/command-installable-value.cc new file mode 100644 index 000000000..d7581534b --- /dev/null +++ b/src/libcmd/command-installable-value.cc @@ -0,0 +1,11 @@ +#include "command-installable-value.hh" + +namespace nix { + +void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable) +{ + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); +} + +} diff --git a/src/libcmd/command-installable-value.hh b/src/libcmd/command-installable-value.hh new file mode 100644 index 000000000..7880d4119 --- /dev/null +++ b/src/libcmd/command-installable-value.hh @@ -0,0 +1,23 @@ +#pragma once +///@file + +#include "installable-value.hh" +#include "command.hh" + +namespace nix { + +/** + * An InstallableCommand where the single positional argument must be an + * InstallableValue in particular. + */ +struct InstallableValueCommand : InstallableCommand +{ + /** + * Entry point to this command + */ + virtual void run(ref<Store> store, ref<InstallableValue> installable) = 0; + + void run(ref<Store> store, ref<Installable> installable) override; +}; + +} diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 517cdf617..bedf11e2c 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -4,6 +4,7 @@ #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" +#include "repl.hh" #include <nlohmann/json.hpp> @@ -121,7 +122,7 @@ ref<EvalState> EvalCommand::getEvalState() ; if (startReplOnEvalErrors) { - evalState->debugRepl = &runRepl; + evalState->debugRepl = &AbstractNixRepl::runSimple; }; } return ref<EvalState>(evalState); @@ -164,7 +165,7 @@ BuiltPathsCommand::BuiltPathsCommand(bool recursive) }); } -void BuiltPathsCommand::run(ref<Store> store) +void BuiltPathsCommand::run(ref<Store> store, Installables && installables) { BuiltPaths paths; if (all) { @@ -210,7 +211,7 @@ void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths) run(store, std::move(sorted)); } -void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths) +void StorePathCommand::run(ref<Store> store, StorePaths && storePaths) { if (storePaths.size() != 1) throw UsageError("this command requires exactly one store path"); @@ -218,20 +219,6 @@ void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePath run(store, *storePaths.begin()); } -Strings editorFor(const Path & file, uint32_t line) -{ - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString<Strings>(editor); - if (line > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos || - editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(file); - return args; -} - MixProfile::MixProfile() { addFlag({ @@ -259,7 +246,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) { if (!profile) return; - std::vector<StorePath> result; + StorePaths result; for (auto & buildable : buildables) { std::visit(overloaded { diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index d16bdbc4b..96236b987 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "installables.hh" +#include "installable-value.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" @@ -18,17 +19,21 @@ class EvalState; struct Pos; class Store; +static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; static constexpr Command::Category catUtility = 101; static constexpr Command::Category catNixInstallation = 102; -static constexpr auto installablesCategory = "Options that change the interpretation of installables"; +static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)"; struct NixMultiCommand : virtual MultiCommand, virtual Command { nlohmann::json toJSON() override; }; +// For the overloaded run methods +#pragma GCC diagnostic ignored "-Woverloaded-virtual" + /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { @@ -94,14 +99,13 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional<Path> file; std::optional<std::string> expr; - bool readOnlyMode = false; - SourceExprCommand(bool supportReadOnlyMode = false); + SourceExprCommand(); - std::vector<std::shared_ptr<Installable>> parseInstallables( + Installables parseInstallables( ref<Store> store, std::vector<std::string> ss); - std::shared_ptr<Installable> parseInstallable( + ref<Installable> parseInstallable( ref<Store> store, const std::string & installable); virtual Strings getDefaultFlakeAttrPaths(); @@ -111,34 +115,48 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions void completeInstallable(std::string_view prefix); }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ -struct InstallablesCommand : virtual Args, SourceExprCommand +struct MixReadOnlyOption : virtual Args +{ + MixReadOnlyOption(); +}; + +/* Like InstallablesCommand but the installables are not loaded */ +struct RawInstallablesCommand : virtual Args, SourceExprCommand { - std::vector<std::shared_ptr<Installable>> installables; + RawInstallablesCommand(); - InstallablesCommand(); + virtual void run(ref<Store> store, std::vector<std::string> && rawInstallables) = 0; - void prepare() override; - Installables load(); + void run(ref<Store> store) override; + + // FIXME make const after CmdRepl's override is fixed up + virtual void applyDefaultInstallables(std::vector<std::string> & rawInstallables); - virtual bool useDefaultInstallables() { return true; } + bool readFromStdIn = false; std::vector<std::string> getFlakesForCompletion() override; -protected: +private: - std::vector<std::string> _installables; + std::vector<std::string> rawInstallables; +}; +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : RawInstallablesCommand +{ + virtual void run(ref<Store> store, Installables && installables) = 0; + + void run(ref<Store> store, std::vector<std::string> && rawInstallables) override; }; /* A command that operates on exactly one "installable" */ struct InstallableCommand : virtual Args, SourceExprCommand { - std::shared_ptr<Installable> installable; + InstallableCommand(); - InstallableCommand(bool supportReadOnlyMode = false); + virtual void run(ref<Store> store, ref<Installable> installable) = 0; - void prepare() override; + void run(ref<Store> store) override; std::vector<std::string> getFlakesForCompletion() override { @@ -173,22 +191,18 @@ public: BuiltPathsCommand(bool recursive = false); - using StoreCommand::run; - virtual void run(ref<Store> store, BuiltPaths && paths) = 0; - void run(ref<Store> store) override; + void run(ref<Store> store, Installables && installables) override; - bool useDefaultInstallables() override { return !all; } + void applyDefaultInstallables(std::vector<std::string> & rawInstallables) override; }; struct StorePathsCommand : public BuiltPathsCommand { StorePathsCommand(bool recursive = false); - using BuiltPathsCommand::run; - - virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0; + virtual void run(ref<Store> store, StorePaths && storePaths) = 0; void run(ref<Store> store, BuiltPaths && paths) override; }; @@ -196,11 +210,9 @@ struct StorePathsCommand : public BuiltPathsCommand /* A command that operates on exactly one store path. */ struct StorePathCommand : public StorePathsCommand { - using StorePathsCommand::run; - virtual void run(ref<Store> store, const StorePath & storePath) = 0; - void run(ref<Store> store, std::vector<StorePath> && storePaths) override; + void run(ref<Store> store, StorePaths && storePaths) override; }; /* A helper class for registering commands globally. */ @@ -231,10 +243,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name) return RegisterCommand(std::move(name), [](){ return make_ref<T>(); }); } -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ -Strings editorFor(const Path & file, uint32_t line); - struct MixProfile : virtual StoreCommand { std::optional<Path> profile; @@ -284,8 +292,4 @@ void printClosureDiff( const StorePath & afterPath, std::string_view indent); - -void runRepl( - ref<EvalState> evalState, - const ValMap & extraEnv); } diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 908127b4d..5b6477c82 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -136,7 +136,11 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "eval-store", - .description = "The Nix store to use for evaluations.", + .description = + R"( + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. + )", .category = category, .labels = {"store-url"}, .handler = {&evalStoreUrl}, @@ -166,7 +170,7 @@ Path lookupFileArg(EvalState & state, std::string_view s) } else if (hasPrefix(s, "flake:")) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; return state.store->toRealPath(storePath); diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 1ec800613..b69db11dd 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "args.hh" diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc new file mode 100644 index 000000000..f674f32bd --- /dev/null +++ b/src/libcmd/editor-for.cc @@ -0,0 +1,20 @@ +#include "util.hh" +#include "editor-for.hh" + +namespace nix { + +Strings editorFor(const Path & file, uint32_t line) +{ + auto editor = getEnv("EDITOR").value_or("cat"); + auto args = tokenizeString<Strings>(editor); + if (line > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos || + editor.find("kak") != std::string::npos)) + args.push_back(fmt("+%d", line)); + args.push_back(file); + return args; +} + +} diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh new file mode 100644 index 000000000..f752bd849 --- /dev/null +++ b/src/libcmd/editor-for.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { + +/* Helper function to generate args that invoke $EDITOR on + filename:lineno. */ +Strings editorFor(const Path & file, uint32_t line); + +} diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc new file mode 100644 index 000000000..cf513126d --- /dev/null +++ b/src/libcmd/installable-attr-path.cc @@ -0,0 +1,113 @@ +#include "globals.hh" +#include "installable-attr-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +InstallableAttrPath::InstallableAttrPath( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec) + : InstallableValue(state) + , cmd(cmd) + , v(allocRootValue(v)) + , attrPath(attrPath) + , extendedOutputsSpec(std::move(extendedOutputsSpec)) +{ } + +std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state) +{ + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes, pos); + return {vRes, pos}; +} + +DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() +{ + auto v = toValue(*state).first; + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map<StorePath, OutputsSpec> byDrvPath; + + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); + + auto newOutputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + for (auto & output : drvInfo.queryOutputs(false, true)) + outputsToInstall.insert(output.first); + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()); + + auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); + + if (!didInsert) + iter->second = iter->second.union_(newOutputs); + } + + DerivedPathsWithInfo res; + for (auto & [drvPath, outputs] : byDrvPath) + res.push_back({ + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = outputs, + }, + .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value { + /* FIXME: reconsider backwards compatibility above + so we can fill in this info. */ + }), + }); + + return res; +} + +InstallableAttrPath InstallableAttrPath::parse( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + return { + state, cmd, v, + prefix == "." ? "" : std::string { prefix }, + extendedOutputsSpec + }; +} + +} diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh new file mode 100644 index 000000000..e9f0c33da --- /dev/null +++ b/src/libcmd/installable-attr-path.hh @@ -0,0 +1,59 @@ +#pragma once +///@file + +#include "globals.hh" +#include "installable-value.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +class InstallableAttrPath : public InstallableValue +{ + SourceExprCommand & cmd; + RootValue v; + std::string attrPath; + ExtendedOutputsSpec extendedOutputsSpec; + + InstallableAttrPath( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec); + + std::string what() const override { return attrPath; }; + + std::pair<Value *, PosIdx> toValue(EvalState & state) override; + + DerivedPathsWithInfo toDerivedPaths() override; + +public: + + static InstallableAttrPath parse( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index a9921b901..6ecf54b7c 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -10,7 +10,10 @@ std::string InstallableDerivedPath::what() const DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() { - return {{.path = derivedPath, .info = {} }}; + return {{ + .path = derivedPath, + .info = make_ref<ExtraPathInfo>(), + }}; } std::optional<StorePath> InstallableDerivedPath::getStorePath() @@ -31,27 +34,24 @@ InstallableDerivedPath InstallableDerivedPath::parse( ExtendedOutputsSpec extendedOutputsSpec) { auto derivedPath = std::visit(overloaded { - // If the user did not use ^, we treat the output more liberally. + // If the user did not use ^, we treat the output more + // liberally: we accept a symlink chain or an actual + // store path. [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { - // First, we accept a symlink chain or an actual store path. auto storePath = store->followLinksToStorePath(prefix); - // Second, we see if the store path ends in `.drv` to decide what sort - // of derived path they want. - // - // This handling predates the `^` syntax. The `^*` in - // `/nix/store/hash-foo.drv^*` unambiguously means "do the - // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could - // also unambiguously mean "do the DerivedPath::Opaque` case". - // - // Issue #7261 tracks reconsidering this `.drv` dispatching. - return storePath.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = std::move(storePath), - .outputs = OutputsSpec::All {}, - } - : (DerivedPath) DerivedPath::Opaque { - .path = std::move(storePath), + // Remove this prior to stabilizing the new CLI. + if (storePath.isDerivation()) { + auto oldDerivedPath = DerivedPath::Built { + .drvPath = storePath, + .outputs = OutputsSpec::All { }, }; + warn( + "The interpretation of store paths arguments ending in `.drv` recently changed. If this command is now failing try again with '%s'", + oldDerivedPath.to_string(*store)); + }; + return DerivedPath::Opaque { + .path = std::move(storePath), + }; }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh index 042878b91..e0b4f18b3 100644 --- a/src/libcmd/installable-derived-path.hh +++ b/src/libcmd/installable-derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "installables.hh" diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc new file mode 100644 index 000000000..a3352af76 --- /dev/null +++ b/src/libcmd/installable-flake.cc @@ -0,0 +1,241 @@ +#include "globals.hh" +#include "installable-flake.hh" +#include "installable-derived-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +std::vector<std::string> InstallableFlake::getActualAttrPaths() +{ + std::vector<std::string> res; + + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); + + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); + + return aOutputs->value; +} + +static std::string showAttrPaths(const std::vector<std::string> & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; + } + return s; +} + +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref<EvalState> state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), + prefixes(fragment == "" ? Strings{} : prefixes), + extendedOutputsSpec(std::move(extendedOutputsSpec)), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() +{ + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + + auto attr = getCursor(*state); + + auto attrPath = attr->getAttrPathStr(); + + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (v.type() == nPath) { + PathSet context; + auto storePath = state->copyPathToStore(context, Path(v.path)); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + }, + .info = make_ref<ExtraPathInfo>(), + }}; + } + + else if (v.type() == nString) { + PathSet context; + auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); + auto storePath = state->store->maybeParseStorePath(s); + if (storePath && context.count(std::string(s))) { + return {{ + .path = DerivedPath::Opaque { + .path = std::move(*storePath), + }, + .info = make_ref<ExtraPathInfo>(), + }}; + } else + throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); + } + + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); + } + + auto drvPath = attr->forceDerivation(); + + std::optional<NixInt> priority; + + if (attr->maybeGetAttr(state->sOutputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } + + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (aOutputSpecified->getBool()) { + if (auto aOutputName = attr->maybeGetAttr("outputName")) + outputsToInstall = { aOutputName->getString() }; + } + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) + for (auto & s : aOutputsToInstall->getListOfStrings()) + outputsToInstall.insert(s); + } + + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); + + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()), + }, + .info = make_ref<ExtraPathInfoFlake>( + ExtraPathInfoValue::Value { + .priority = priority, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + }, + ExtraPathInfoFlake::Flake { + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + }), + }}; +} + +std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) +{ + return {&getCursor(state)->forceValue(), noPos}; +} + +std::vector<ref<eval_cache::AttrCursor>> +InstallableFlake::getCursors(EvalState & state) +{ + auto evalCache = openEvalCache(state, getLockedFlake()); + + auto root = evalCache->getRoot(); + + std::vector<ref<eval_cache::AttrCursor>> res; + + Suggestions suggestions; + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); + } + } + + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; +} + +std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + // FIXME why this side effect? + lockFlagsApplyConfig.applyNixConfig = true; + _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + } + return _lockedFlake; +} + +FlakeRef InstallableFlake::nixpkgsFlakeRef() const +{ + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } + } + + return InstallableValue::nixpkgsFlakeRef(); +} + +} diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh new file mode 100644 index 000000000..afe64d977 --- /dev/null +++ b/src/libcmd/installable-flake.hh @@ -0,0 +1,77 @@ +#pragma once +///@file + +#include "installable-value.hh" + +namespace nix { + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Flake. + * + * Invariant: every ExtraPathInfo gotten from an InstallableFlake should + * be possible to downcast to an ExtraPathInfoFlake. + */ +struct ExtraPathInfoFlake : ExtraPathInfoValue +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Flake { + FlakeRef originalRef; + FlakeRef resolvedRef; + }; + + Flake flake; + + ExtraPathInfoFlake(Value && v, Flake && f) + : ExtraPathInfoValue(std::move(v)), flake(f) + { } +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + ExtendedOutputsSpec extendedOutputsSpec; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; + + InstallableFlake( + SourceExprCommand * cmd, + ref<EvalState> state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags); + + std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector<std::string> getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + DerivedPathsWithInfo toDerivedPaths() override; + + std::pair<Value *, PosIdx> toValue(EvalState & state) override; + + /** + * Get a cursor to every attrpath in getActualAttrPaths() that + * exists. However if none exists, throw an exception. + */ + std::vector<ref<eval_cache::AttrCursor>> + getCursors(EvalState & state) override; + + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const override; +}; + +ref<eval_cache::EvalCache> openEvalCache( + EvalState & state, + std::shared_ptr<flake::LockedFlake> lockedFlake); + +} diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc new file mode 100644 index 000000000..30f80edb2 --- /dev/null +++ b/src/libcmd/installable-value.cc @@ -0,0 +1,44 @@ +#include "installable-value.hh" +#include "eval-cache.hh" + +namespace nix { + +std::vector<ref<eval_cache::AttrCursor>> +InstallableValue::getCursors(EvalState & state) +{ + auto evalCache = + std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state, + [&]() { return toValue(state).first; }); + return {evalCache->getRoot()}; +} + +ref<eval_cache::AttrCursor> +InstallableValue::getCursor(EvalState & state) +{ + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); +} + +static UsageError nonValueInstallable(Installable & installable) +{ + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); +} + +InstallableValue & InstallableValue::require(Installable & installable) +{ + auto * castedInstallable = dynamic_cast<InstallableValue *>(&installable); + if (!castedInstallable) + throw nonValueInstallable(installable); + return *castedInstallable; +} + +ref<InstallableValue> InstallableValue::require(ref<Installable> installable) +{ + auto castedInstallable = installable.dynamic_pointer_cast<InstallableValue>(); + if (!castedInstallable) + throw nonValueInstallable(*installable); + return ref { castedInstallable }; +} + +} diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh new file mode 100644 index 000000000..bfb3bfeed --- /dev/null +++ b/src/libcmd/installable-value.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include "installables.hh" +#include "flake/flake.hh" + +namespace nix { + +struct DrvInfo; +struct SourceExprCommand; + +namespace eval_cache { class EvalCache; class AttrCursor; } + +struct App +{ + std::vector<DerivedPath> context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... +}; + +struct UnresolvedApp +{ + App unresolved; + App resolve(ref<Store> evalStore, ref<Store> store); +}; + +/** + * Extra info about a \ref DerivedPath "derived path" that ultimately + * come from a Nix language value. + * + * Invariant: every ExtraPathInfo gotten from an InstallableValue should + * be possible to downcast to an ExtraPathInfoValue. + */ +struct ExtraPathInfoValue : ExtraPathInfo +{ + /** + * Extra struct to get around C++ designated initializer limitations + */ + struct Value { + /** + * An optional priority for use with "build envs". See Package + */ + std::optional<NixInt> priority; + + /** + * The attribute path associated with this value. The idea is + * that an installable referring to a value typically refers to + * a larger value, from which we project a smaller value out + * with this. + */ + std::string attrPath; + + /** + * \todo merge with DerivedPath's 'outputs' field? + */ + ExtendedOutputsSpec extendedOutputsSpec; + }; + + Value value; + + ExtraPathInfoValue(Value && v) + : value(v) + { } + + virtual ~ExtraPathInfoValue() = default; +}; + +/** + * An Installable which corresponds a Nix langauge value, in addition to + * a collection of \ref DerivedPath "derived paths". + */ +struct InstallableValue : Installable +{ + ref<EvalState> state; + + InstallableValue(ref<EvalState> state) : state(state) {} + + virtual ~InstallableValue() { } + + virtual std::pair<Value *, PosIdx> toValue(EvalState & state) = 0; + + /** + * Get a cursor to each value this Installable could refer to. + * However if none exists, throw exception instead of returning + * empty vector. + */ + virtual std::vector<ref<eval_cache::AttrCursor>> + getCursors(EvalState & state); + + /** + * Get the first and most preferred cursor this Installable could + * refer to, or throw an exception if none exists. + */ + virtual ref<eval_cache::AttrCursor> + getCursor(EvalState & state); + + UnresolvedApp toApp(EvalState & state); + + virtual FlakeRef nixpkgsFlakeRef() const + { + return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + } + + static InstallableValue & require(Installable & installable); + static ref<InstallableValue> require(ref<Installable> installable); +}; + +} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 85edc28cf..32ae46d9f 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,6 +1,8 @@ #include "globals.hh" #include "installables.hh" #include "installable-derived-path.hh" +#include "installable-attr-path.hh" +#include "installable-flake.hh" #include "outputs-spec.hh" #include "util.hh" #include "command.hh" @@ -101,6 +103,28 @@ MixFlakeOptions::MixFlakeOptions() }); addFlag({ + .longName = "reference-lock-file", + .description = "Read the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.referenceLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ + .longName = "output-lock-file", + .description = "Write the given lock file instead of `flake.lock` within the top-level flake.", + .category = category, + .labels = {"flake-lock-path"}, + .handler = {[&](std::string lockFilePath) { + lockFlags.outputLockFilePath = lockFilePath; + }}, + .completer = completePath + }); + + addFlag({ .longName = "inputs-from", .description = "Use the inputs of the specified flake as registry entries.", .category = category, @@ -145,13 +169,13 @@ void MixFlakeOptions::completionHook() completeFlakeInput(*prefix); } -SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) +SourceExprCommand::SourceExprCommand() { addFlag({ .longName = "file", .shortName = 'f', .description = - "Interpret installables as attribute paths relative to the Nix expression stored in *file*. " + "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression stored in *file*. " "If *file* is the character -, then a Nix expression will be read from standard input. " "Implies `--impure`.", .category = installablesCategory, @@ -162,22 +186,23 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) addFlag({ .longName = "expr", - .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", + .description = "Interpret [*installables*](@docroot@/command-ref/new-cli/nix.md#installables) as attribute paths relative to the Nix expression *expr*.", .category = installablesCategory, .labels = {"expr"}, .handler = {&expr} }); +} - if (supportReadOnlyMode) { - addFlag({ - .longName = "read-only", - .description = - "Do not instantiate each evaluated derivation. " - "This improves performance, but can cause errors when accessing " - "store paths of derivations during evaluation.", - .handler = {&readOnlyMode, true}, - }); - } +MixReadOnlyOption::MixReadOnlyOption() +{ + addFlag({ + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&settings.readOnlyMode, true}, + }); } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -329,7 +354,7 @@ void completeFlakeRefWithFragment( void completeFlakeRef(ref<Store> store, std::string_view prefix) { - if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") @@ -361,23 +386,6 @@ DerivedPathWithInfo Installable::toDerivedPath() return std::move(buildables[0]); } -std::vector<ref<eval_cache::AttrCursor>> -Installable::getCursors(EvalState & state) -{ - auto evalCache = - std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state, - [&]() { return toValue(state).first; }); - return {evalCache->getRoot()}; -} - -ref<eval_cache::AttrCursor> -Installable::getCursor(EvalState & state) -{ - /* Although getCursors should return at least one element, in case it doesn't, - bound check to avoid an undefined behavior for vector[0] */ - return getCursors(state).at(0); -} - static StorePath getDeriver( ref<Store> store, const Installable & i, @@ -390,111 +398,6 @@ static StorePath getDeriver( return *derivers.begin(); } -struct InstallableAttrPath : InstallableValue -{ - SourceExprCommand & cmd; - RootValue v; - std::string attrPath; - ExtendedOutputsSpec extendedOutputsSpec; - - InstallableAttrPath( - ref<EvalState> state, - SourceExprCommand & cmd, - Value * v, - const std::string & attrPath, - ExtendedOutputsSpec extendedOutputsSpec) - : InstallableValue(state) - , cmd(cmd) - , v(allocRootValue(v)) - , attrPath(attrPath) - , extendedOutputsSpec(std::move(extendedOutputsSpec)) - { } - - std::string what() const override { return attrPath; } - - std::pair<Value *, PosIdx> toValue(EvalState & state) override - { - auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); - state.forceValue(*vRes, pos); - return {vRes, pos}; - } - - DerivedPathsWithInfo toDerivedPaths() override - { - auto v = toValue(*state).first; - - Bindings & autoArgs = *cmd.getAutoArgs(*state); - - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); - - // Backward compatibility hack: group results by drvPath. This - // helps keep .all output together. - std::map<StorePath, OutputsSpec> byDrvPath; - - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); - - auto newOutputs = std::visit(overloaded { - [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set<std::string> outputsToInstall; - for (auto & output : drvInfo.queryOutputs(false, true)) - outputsToInstall.insert(output.first); - return OutputsSpec::Names { std::move(outputsToInstall) }; - }, - [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { - return e; - }, - }, extendedOutputsSpec.raw()); - - auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); - - if (!didInsert) - iter->second = iter->second.union_(newOutputs); - } - - DerivedPathsWithInfo res; - for (auto & [drvPath, outputs] : byDrvPath) - res.push_back({ - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = outputs, - }, - }); - - return res; - } -}; - -std::vector<std::string> InstallableFlake::getActualAttrPaths() -{ - std::vector<std::string> res; - - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); - - for (auto & s : attrPaths) - res.push_back(s); - - return res; -} - -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); - - return aOutputs->value; -} - ref<eval_cache::EvalCache> openEvalCache( EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake) @@ -524,195 +427,10 @@ ref<eval_cache::EvalCache> openEvalCache( }); } -static std::string showAttrPaths(const std::vector<std::string> & paths) -{ - std::string s; - for (const auto & [n, i] : enumerate(paths)) { - if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; - s += '\''; s += i; s += '\''; - } - return s; -} - -InstallableFlake::InstallableFlake( - SourceExprCommand * cmd, - ref<EvalState> state, - FlakeRef && flakeRef, - std::string_view fragment, - ExtendedOutputsSpec extendedOutputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags) - : InstallableValue(state), - flakeRef(flakeRef), - attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), - prefixes(fragment == "" ? Strings{} : prefixes), - extendedOutputsSpec(std::move(extendedOutputsSpec)), - lockFlags(lockFlags) -{ - if (cmd && cmd->getAutoArgs(*state)->size()) - throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); -} - -DerivedPathsWithInfo InstallableFlake::toDerivedPaths() -{ - Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); - - auto attr = getCursor(*state); - - auto attrPath = attr->getAttrPathStr(); - - if (!attr->isDerivation()) { - - // FIXME: use eval cache? - auto v = attr->forceValue(); - - if (v.type() == nPath) { - PathSet context; - auto storePath = state->copyPathToStore(context, Path(v.path)); - return {{ - .path = DerivedPath::Opaque { - .path = std::move(storePath), - } - }}; - } - - else if (v.type() == nString) { - PathSet context; - auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); - auto storePath = state->store->maybeParseStorePath(s); - if (storePath && context.count(std::string(s))) { - return {{ - .path = DerivedPath::Opaque { - .path = std::move(*storePath), - } - }}; - } else - throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); - } - - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); - } - - auto drvPath = attr->forceDerivation(); - - std::optional<NixInt> priority; - - if (attr->maybeGetAttr(state->sOutputSpecified)) { - } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); - } - - return {{ - .path = DerivedPath::Built { - .drvPath = std::move(drvPath), - .outputs = std::visit(overloaded { - [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set<std::string> outputsToInstall; - if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { - if (aOutputSpecified->getBool()) { - if (auto aOutputName = attr->maybeGetAttr("outputName")) - outputsToInstall = { aOutputName->getString() }; - } - } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) - for (auto & s : aOutputsToInstall->getListOfStrings()) - outputsToInstall.insert(s); - } - - if (outputsToInstall.empty()) - outputsToInstall.insert("out"); - - return OutputsSpec::Names { std::move(outputsToInstall) }; - }, - [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { - return e; - }, - }, extendedOutputsSpec.raw()), - }, - .info = { - .priority = priority, - .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, - .attrPath = attrPath, - .extendedOutputsSpec = extendedOutputsSpec, - } - }}; -} - -std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) -{ - return {&getCursor(state)->forceValue(), noPos}; -} - -std::vector<ref<eval_cache::AttrCursor>> -InstallableFlake::getCursors(EvalState & state) -{ - auto evalCache = openEvalCache(state, - std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - std::vector<ref<eval_cache::AttrCursor>> res; - - Suggestions suggestions; - auto attrPaths = getActualAttrPaths(); - - for (auto & attrPath : attrPaths) { - debug("trying flake output attribute '%s'", attrPath); - - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) { - res.push_back(ref(*attr)); - } else { - suggestions += attr.getSuggestions(); - } - } - - if (res.size() == 0) - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); - - return res; -} - -std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const -{ - if (!_lockedFlake) { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); - } - return _lockedFlake; -} - -FlakeRef InstallableFlake::nixpkgsFlakeRef() const -{ - auto lockedFlake = getLockedFlake(); - - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } - } - - return Installable::nixpkgsFlakeRef(); -} - -std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( +Installables SourceExprCommand::parseInstallables( ref<Store> store, std::vector<std::string> ss) { - std::vector<std::shared_ptr<Installable>> result; - - if (readOnlyMode) { - settings.readOnlyMode = true; - } + Installables result; if (file || expr) { if (file && expr) @@ -738,10 +456,9 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( for (auto & s : ss) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s); result.push_back( - std::make_shared<InstallableAttrPath>( - state, *this, vFile, - prefix == "." ? "" : std::string { prefix }, - extendedOutputsSpec)); + make_ref<InstallableAttrPath>( + InstallableAttrPath::parse( + state, *this, vFile, prefix, extendedOutputsSpec))); } } else { @@ -756,7 +473,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( if (prefix.find('/') != std::string::npos) { try { - result.push_back(std::make_shared<InstallableDerivedPath>( + result.push_back(make_ref<InstallableDerivedPath>( InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); continue; } catch (BadStorePath &) { @@ -768,7 +485,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( try { auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); - result.push_back(std::make_shared<InstallableFlake>( + result.push_back(make_ref<InstallableFlake>( this, getEvalState(), std::move(flakeRef), @@ -789,7 +506,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( return result; } -std::shared_ptr<Installable> SourceExprCommand::parseInstallable( +ref<Installable> SourceExprCommand::parseInstallable( ref<Store> store, const std::string & installable) { auto installables = parseInstallables(store, {installable}); @@ -801,7 +518,7 @@ std::vector<BuiltPathWithResult> Installable::build( ref<Store> evalStore, ref<Store> store, Realise mode, - const std::vector<std::shared_ptr<Installable>> & installables, + const Installables & installables, BuildMode bMode) { std::vector<BuiltPathWithResult> res; @@ -810,11 +527,11 @@ std::vector<BuiltPathWithResult> Installable::build( return res; } -std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Installable::build2( +std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build2( ref<Store> evalStore, ref<Store> store, Realise mode, - const std::vector<std::shared_ptr<Installable>> & installables, + const Installables & installables, BuildMode bMode) { if (mode == Realise::Nothing) @@ -822,8 +539,8 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal struct Aux { - ExtraPathInfo info; - std::shared_ptr<Installable> installable; + ref<ExtraPathInfo> info; + ref<Installable> installable; }; std::vector<DerivedPath> pathsToBuild; @@ -836,7 +553,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal } } - std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> res; + std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> res; switch (mode) { @@ -908,7 +625,7 @@ BuiltPaths Installable::toBuiltPaths( ref<Store> store, Realise mode, OperateOn operateOn, - const std::vector<std::shared_ptr<Installable>> & installables) + const Installables & installables) { if (operateOn == OperateOn::Output) { BuiltPaths res; @@ -930,7 +647,7 @@ StorePathSet Installable::toStorePaths( ref<Store> evalStore, ref<Store> store, Realise mode, OperateOn operateOn, - const std::vector<std::shared_ptr<Installable>> & installables) + const Installables & installables) { StorePathSet outPaths; for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { @@ -944,7 +661,7 @@ StorePath Installable::toStorePath( ref<Store> evalStore, ref<Store> store, Realise mode, OperateOn operateOn, - std::shared_ptr<Installable> installable) + ref<Installable> installable) { auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); @@ -956,7 +673,7 @@ StorePath Installable::toStorePath( StorePathSet Installable::toDerivations( ref<Store> store, - const std::vector<std::shared_ptr<Installable>> & installables, + const Installables & installables, bool useDeriver) { StorePathSet drvPaths; @@ -965,9 +682,12 @@ StorePathSet Installable::toDerivations( for (const auto & b : i->toDerivedPaths()) std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - drvPaths.insert(getDeriver(store, *i, bo.path)); + drvPaths.insert( + bo.path.isDerivation() + ? bo.path + : useDeriver + ? getDeriver(store, *i, bo.path) + : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); @@ -977,43 +697,59 @@ StorePathSet Installable::toDerivations( return drvPaths; } -InstallablesCommand::InstallablesCommand() +RawInstallablesCommand::RawInstallablesCommand() { + addFlag({ + .longName = "stdin", + .description = "Read installables from the standard input.", + .handler = {&readFromStdIn, true} + }); + expectArgs({ .label = "installables", - .handler = {&_installables}, + .handler = {&rawInstallables}, .completer = {[&](size_t, std::string_view prefix) { completeInstallable(prefix); }} }); } -void InstallablesCommand::prepare() +void RawInstallablesCommand::applyDefaultInstallables(std::vector<std::string> & rawInstallables) { - installables = load(); -} - -Installables InstallablesCommand::load() { - Installables installables; - if (_installables.empty() && useDefaultInstallables()) + if (rawInstallables.empty()) { // FIXME: commands like "nix profile install" should not have a // default, probably. - _installables.push_back("."); - return parseInstallables(getStore(), _installables); + rawInstallables.push_back("."); + } } -std::vector<std::string> InstallablesCommand::getFlakesForCompletion() +void RawInstallablesCommand::run(ref<Store> store) { - if (_installables.empty()) { - if (useDefaultInstallables()) - return {"."}; - return {}; + if (readFromStdIn && !isatty(STDIN_FILENO)) { + std::string word; + while (std::cin >> word) { + rawInstallables.emplace_back(std::move(word)); + } } - return _installables; + + applyDefaultInstallables(rawInstallables); + run(store, std::move(rawInstallables)); } -InstallableCommand::InstallableCommand(bool supportReadOnlyMode) - : SourceExprCommand(supportReadOnlyMode) +std::vector<std::string> RawInstallablesCommand::getFlakesForCompletion() +{ + applyDefaultInstallables(rawInstallables); + return rawInstallables; +} + +void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables) +{ + auto installables = parseInstallables(store, rawInstallables); + run(store, std::move(installables)); +} + +InstallableCommand::InstallableCommand() + : SourceExprCommand() { expectArgs({ .label = "installable", @@ -1025,9 +761,16 @@ InstallableCommand::InstallableCommand(bool supportReadOnlyMode) }); } -void InstallableCommand::prepare() +void InstallableCommand::run(ref<Store> store) +{ + auto installable = parseInstallable(store, _installable); + run(store, std::move(installable)); +} + +void BuiltPathsCommand::applyDefaultInstallables(std::vector<std::string> & rawInstallables) { - installable = parseInstallable(getStore(), _installable); + if (rawInstallables.empty() && !all) + rawInstallables.push_back("."); } } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index da6a3addd..42d6c7c7c 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,12 +1,11 @@ #pragma once +///@file #include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" -#include "eval.hh" #include "store-api.hh" -#include "flake/flake.hh" #include "build-result.hh" #include <optional> @@ -14,122 +13,156 @@ namespace nix { struct DrvInfo; -struct SourceExprCommand; - -namespace eval_cache { class EvalCache; class AttrCursor; } - -struct App -{ - std::vector<DerivedPath> context; - Path program; - // FIXME: add args, sandbox settings, metadata, ... -}; - -struct UnresolvedApp -{ - App unresolved; - App resolve(ref<Store> evalStore, ref<Store> store); -}; enum class Realise { - /* Build the derivation. Postcondition: the - derivation outputs exist. */ + /** + * Build the derivation. + * + * Postcondition: the derivation outputs exist. + */ Outputs, - /* Don't build the derivation. Postcondition: the store derivation - exists. */ + /** + * Don't build the derivation. + * + * Postcondition: the store derivation exists. + */ Derivation, - /* Evaluate in dry-run mode. Postcondition: nothing. */ - // FIXME: currently unused, but could be revived if we can - // evaluate derivations in-memory. + /** + * Evaluate in dry-run mode. + * + * Postcondition: nothing. + * + * \todo currently unused, but could be revived if we can evaluate + * derivations in-memory. + */ Nothing }; -/* How to handle derivations in commands that operate on store paths. */ +/** + * How to handle derivations in commands that operate on store paths. + */ enum class OperateOn { - /* Operate on the output path. */ + /** + * Operate on the output path. + */ Output, - /* Operate on the .drv path. */ + /** + * Operate on the .drv path. + */ Derivation }; +/** + * Extra info about a DerivedPath + * + * Yes, this is empty, but that is intended. It will be sub-classed by + * the subclasses of Installable to allow those to provide more info. + * Certain commands will make use of this info. + */ struct ExtraPathInfo { - std::optional<NixInt> priority; - std::optional<FlakeRef> originalRef; - std::optional<FlakeRef> resolvedRef; - std::optional<std::string> attrPath; - // FIXME: merge with DerivedPath's 'outputs' field? - std::optional<ExtendedOutputsSpec> extendedOutputsSpec; + virtual ~ExtraPathInfo() = default; }; -/* A derived path with any additional info that commands might - need from the derivation. */ +/** + * A DerivedPath with \ref ExtraPathInfo "any additional info" that + * commands might need from the derivation. + */ struct DerivedPathWithInfo { DerivedPath path; - ExtraPathInfo info; + ref<ExtraPathInfo> info; }; +/** + * Like DerivedPathWithInfo but extending BuiltPath with \ref + * ExtraPathInfo "extra info" and also possibly the \ref BuildResult + * "result of building". + */ struct BuiltPathWithResult { BuiltPath path; - ExtraPathInfo info; + ref<ExtraPathInfo> info; std::optional<BuildResult> result; }; +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo; +struct Installable; + +/** + * Shorthand, for less typing and helping us keep the choice of + * collection in sync. + */ +typedef std::vector<ref<Installable>> Installables; + +/** + * Installables are the main positional arguments for the Nix + * Command-line. + * + * This base class is very flexible, and just assumes and the + * Installable refers to a collection of \ref DerivedPath "derived paths" with + * \ref ExtraPathInfo "extra info". + */ struct Installable { virtual ~Installable() { } + /** + * What Installable is this? + * + * Prints back valid CLI syntax that would result in this same + * installable. It doesn't need to be exactly what the user wrote, + * just something that means the same thing. + */ virtual std::string what() const = 0; + /** + * Get the collection of \ref DerivedPathWithInfo "derived paths + * with info" that this \ref Installable instalallable denotes. + * + * This is the main method of this class + */ virtual DerivedPathsWithInfo toDerivedPaths() = 0; + /** + * A convenience wrapper of the above for when we expect an + * installable to produce a single \ref DerivedPath "derived path" + * only. + * + * If no or multiple \ref DerivedPath "derived paths" are produced, + * and error is raised. + */ DerivedPathWithInfo toDerivedPath(); - UnresolvedApp toApp(EvalState & state); - - virtual std::pair<Value *, PosIdx> toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } - - /* Return a value only if this installable is a store path or a - symlink to it. */ + /** + * Return a value only if this installable is a store path or a + * symlink to it. + * + * \todo should we move this to InstallableDerivedPath? It is only + * supposed to work there anyways. Can always downcast. + */ virtual std::optional<StorePath> getStorePath() { return {}; } - /* Get a cursor to each value this Installable could refer to. However - if none exists, throw exception instead of returning empty vector. */ - virtual std::vector<ref<eval_cache::AttrCursor>> - getCursors(EvalState & state); - - /* Get the first and most preferred cursor this Installable could refer - to, or throw an exception if none exists. */ - virtual ref<eval_cache::AttrCursor> - getCursor(EvalState & state); - - virtual FlakeRef nixpkgsFlakeRef() const - { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); - } - static std::vector<BuiltPathWithResult> build( ref<Store> evalStore, ref<Store> store, Realise mode, - const std::vector<std::shared_ptr<Installable>> & installables, + const Installables & installables, BuildMode bMode = bmNormal); - static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> build2( + static std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> build2( ref<Store> evalStore, ref<Store> store, Realise mode, - const std::vector<std::shared_ptr<Installable>> & installables, + const Installables & installables, BuildMode bMode = bmNormal); static std::set<StorePath> toStorePaths( @@ -137,18 +170,18 @@ struct Installable ref<Store> store, Realise mode, OperateOn operateOn, - const std::vector<std::shared_ptr<Installable>> & installables); + const Installables & installables); static StorePath toStorePath( ref<Store> evalStore, ref<Store> store, Realise mode, OperateOn operateOn, - std::shared_ptr<Installable> installable); + ref<Installable> installable); static std::set<StorePath> toDerivations( ref<Store> store, - const std::vector<std::shared_ptr<Installable>> & installables, + const Installables & installables, bool useDeriver = false); static BuiltPaths toBuiltPaths( @@ -156,59 +189,7 @@ struct Installable ref<Store> store, Realise mode, OperateOn operateOn, - const std::vector<std::shared_ptr<Installable>> & installables); -}; - -typedef std::vector<std::shared_ptr<Installable>> Installables; - -struct InstallableValue : Installable -{ - ref<EvalState> state; - - InstallableValue(ref<EvalState> state) : state(state) {} + const Installables & installables); }; -struct InstallableFlake : InstallableValue -{ - FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; - ExtendedOutputsSpec extendedOutputsSpec; - const flake::LockFlags & lockFlags; - mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; - - InstallableFlake( - SourceExprCommand * cmd, - ref<EvalState> state, - FlakeRef && flakeRef, - std::string_view fragment, - ExtendedOutputsSpec extendedOutputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags); - - std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } - - std::vector<std::string> getActualAttrPaths(); - - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - - DerivedPathsWithInfo toDerivedPaths() override; - - std::pair<Value *, PosIdx> toValue(EvalState & state) override; - - /* Get a cursor to every attrpath in getActualAttrPaths() - that exists. However if none exists, throw an exception. */ - std::vector<ref<eval_cache::AttrCursor>> - getCursors(EvalState & state) override; - - std::shared_ptr<flake::LockedFlake> getLockedFlake() const; - - FlakeRef nixpkgsFlakeRef() const override; -}; - -ref<eval_cache::EvalCache> openEvalCache( - EvalState & state, - std::shared_ptr<flake::LockedFlake> lockedFlake); - } diff --git a/src/libcmd/legacy.hh b/src/libcmd/legacy.hh index f503b0da3..357500a4d 100644 --- a/src/libcmd/legacy.hh +++ b/src/libcmd/legacy.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <functional> #include <map> diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 152bc388d..541a7d2ba 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,7 +6,7 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix +libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index 78320fcf5..a04d32a4f 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4158439b6..57848a5d3 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -19,6 +19,8 @@ extern "C" { } #endif +#include "repl.hh" + #include "ansicolor.hh" #include "shared.hh" #include "eval.hh" @@ -31,7 +33,9 @@ extern "C" { #include "get-drvs.hh" #include "derivations.hh" #include "globals.hh" -#include "command.hh" +#include "flake/flake.hh" +#include "flake/lockfile.hh" +#include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" @@ -45,18 +49,16 @@ extern "C" { namespace nix { struct NixRepl + : AbstractNixRepl #if HAVE_BOEHMGC - : gc + , gc #endif { std::string curDir; - ref<EvalState> state; - Bindings * autoArgs; size_t debugTraceIndex; Strings loadedFiles; - typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; std::function<AnnotatedValues()> getValues; const static int envSize = 32768; @@ -69,8 +71,11 @@ struct NixRepl NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state, std::function<AnnotatedValues()> getValues); - ~NixRepl(); - void mainLoop(); + virtual ~NixRepl(); + + void mainLoop() override; + void initEnv() override; + StringSet completePrefix(const std::string & prefix); bool getLine(std::string & input, const std::string & prompt); StorePath getDerivationPath(Value & v); @@ -78,7 +83,6 @@ struct NixRepl void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); - void initEnv(); void loadFiles(); void reloadFiles(); void addAttrsToScope(Value & attrs); @@ -92,7 +96,6 @@ struct NixRepl std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); }; - std::string removeWhitespace(std::string s) { s = chomp(s); @@ -104,7 +107,7 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, std::function<NixRepl::AnnotatedValues()> getValues) - : state(state) + : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(false, state->staticBaseEnv.get())) @@ -249,7 +252,9 @@ void NixRepl::mainLoop() el_hist_size = 1000; #endif read_history(historyFile.c_str()); + auto oldRepl = curRepl; curRepl = this; + Finally restoreRepl([&] { curRepl = oldRepl; }); #ifndef READLINE rl_set_complete_func(completionCallback); rl_set_list_possib_func(listPossibleCallback); @@ -1021,6 +1026,8 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << v.fpoint; break; + case nThunk: + case nExternal: default: str << ANSI_RED "«unknown»" ANSI_NORMAL; break; @@ -1029,8 +1036,22 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m return str; } -void runRepl( - ref<EvalState>evalState, + +std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create( + const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + std::function<AnnotatedValues()> getValues) +{ + return std::make_unique<NixRepl>( + searchPath, + openStore(), + state, + getValues + ); +} + + +void AbstractNixRepl::runSimple( + ref<EvalState> evalState, const ValMap & extraEnv) { auto getValues = [&]()->NixRepl::AnnotatedValues{ @@ -1054,91 +1075,4 @@ void runRepl( repl->mainLoop(); } -struct CmdRepl : InstallablesCommand -{ - CmdRepl() { - evalSettings.pureEval = false; - } - - void prepare() override - { - if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { - warn("future versions of Nix will require using `--file` to load a file"); - if (this->_installables.size() > 1) - warn("more than one input file is not currently supported"); - auto filePath = this->_installables[0].data(); - file = std::optional(filePath); - _installables.front() = _installables.back(); - _installables.pop_back(); - } - installables = InstallablesCommand::load(); - } - - std::vector<std::string> files; - - Strings getDefaultFlakeAttrPaths() override - { - return {""}; - } - - bool useDefaultInstallables() override - { - return file.has_value() or expr.has_value(); - } - - bool forceImpureByDefault() override - { - return true; - } - - std::string description() override - { - return "start an interactive environment for evaluating Nix expressions"; - } - - std::string doc() override - { - return - #include "repl.md" - ; - } - - void run(ref<Store> store) override - { - auto state = getEvalState(); - auto getValues = [&]()->NixRepl::AnnotatedValues{ - auto installables = load(); - NixRepl::AnnotatedValues values; - for (auto & installable: installables){ - auto what = installable->what(); - if (file){ - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); - state->forceValue(*val, pos); - auto autoArgs = getAutoArgs(*state); - auto valPost = state->allocValue(); - state->autoCallFunction(*autoArgs, *val, *valPost); - state->forceValue(*valPost, pos); - values.push_back( {valPost, what }); - } else { - auto [val, pos] = installable->toValue(*state); - values.push_back( {val, what} ); - } - } - return values; - }; - auto repl = std::make_unique<NixRepl>( - searchPath, - openStore(), - state, - getValues - ); - repl->autoArgs = getAutoArgs(*repl->state); - repl->initEnv(); - repl->mainLoop(); - } -}; - -static auto rCmdRepl = registerCommand<CmdRepl>("repl"); - } diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh new file mode 100644 index 000000000..731c8e6db --- /dev/null +++ b/src/libcmd/repl.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include "eval.hh" + +#if HAVE_BOEHMGC +#define GC_INCLUDE_NEW +#include <gc/gc_cpp.h> +#endif + +namespace nix { + +struct AbstractNixRepl +{ + ref<EvalState> state; + Bindings * autoArgs; + + AbstractNixRepl(ref<EvalState> state) + : state(state) + { } + + virtual ~AbstractNixRepl() + { } + + typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; + + static std::unique_ptr<AbstractNixRepl> create( + const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + std::function<AnnotatedValues()> getValues); + + static void runSimple( + ref<EvalState> evalState, + const ValMap & extraEnv); + + virtual void initEnv() = 0; + + virtual void mainLoop() = 0; +}; + +} diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 117e0051b..d0d05b1a1 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index dcc73b506..3fe54408b 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "symbol-table.hh" diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c93e55b93..c90882edc 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "hash.hh" diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f0da688db..f8ddd2acc 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3e37c7f60..7f2065656 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -8,6 +8,7 @@ #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" +#include "profiles.hh" #include <algorithm> #include <chrono> @@ -172,7 +173,17 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, case tFloat: str << fpoint; break; + case tBlackhole: + // Although we know for sure that it's going to be an infinite recursion + // when this value is accessed _in the current context_, it's likely + // that the user will misinterpret a simpler «infinite recursion» output + // as a definitive statement about the value, while in fact it may be + // a valid value after `builtins.trace` and perhaps some other steps + // have completed. + str << "«potential infinite recursion»"; + break; default: + printError("Nix evaluator internal error: Value::print(): invalid value type %1%", internalType); abort(); } } @@ -228,6 +239,9 @@ std::string_view showType(ValueType type) std::string showType(const Value & v) { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: @@ -241,16 +255,21 @@ std::string showType(const Value & v) default: return std::string(showType(v.type())); } + #pragma GCC diagnostic pop } PosIdx Value::determinePos(const PosIdx pos) const { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (internalType) { case tAttrs: return attrs->pos; case tLambda: return lambda.fun->pos; case tApp: return app.left->determinePos(pos); default: return pos; } + #pragma GCC diagnostic pop } bool Value::isTrivial() const @@ -368,7 +387,7 @@ void initGC() size = (pageSize * pages) / 4; // 25% of RAM if (size > maxSize) size = maxSize; #endif - debug(format("setting initial heap size to %1% bytes") % size); + debug("setting initial heap size to %1% bytes", size); GC_expand_hp(size); } @@ -519,7 +538,6 @@ EvalState::EvalState( static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); /* Initialise the Nix expression search path. */ - evalSettings.nixPath.setDefault(evalSettings.getDefaultNixPath()); if (!evalSettings.pureEval) { for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); @@ -610,7 +628,7 @@ Path EvalState::checkSourcePath(const Path & path_) } /* Resolve symlinks. */ - debug(format("checking access to '%s'") % abspath); + debug("checking access to '%s'", abspath); Path path = canonPath(abspath, true); for (auto & i : *allowedPaths) { @@ -2327,6 +2345,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nFloat: return v1.fpoint == v2.fpoint; + case nThunk: // Must not be left by forceValue default: error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow<EvalError>(); } @@ -2473,35 +2492,30 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { EvalSettings::EvalSettings() { + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); } -/* impure => NIX_PATH or a default path - * restrict-eval => NIX_PATH - * pure-eval => empty - */ Strings EvalSettings::getDefaultNixPath() { - if (pureEval) - return {}; - - auto var = getEnv("NIX_PATH"); - if (var) { - return parseNixPath(*var); - } else if (restrictEval) { - return {}; - } else { - Strings res; - auto add = [&](const Path & p, const std::optional<std::string> & s = std::nullopt) { - if (pathExists(p)) - res.push_back(s ? *s + "=" + p : p); - }; + Strings res; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathExists(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + if (!evalSettings.restrictEval && !evalSettings.pureEval) { add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); - add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); - add(settings.nixStateDir + "/profiles/per-user/root/channels"); - - return res; + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); } + + return res; } bool EvalSettings::isPseudoUrl(std::string_view s) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 2340ef67b..a1b54951e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "attr-set.hh" #include "types.hh" @@ -570,7 +571,7 @@ struct EvalSettings : Config { EvalSettings(); - Strings getDefaultNixPath(); + static Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); @@ -580,15 +581,8 @@ struct EvalSettings : Config "Whether builtin functions that allow executing native code should be enabled."}; Setting<Strings> nixPath{ - this, {}, "nix-path", - R"( - List of directories to be searched for `<...>` file references. - - If [pure evaluation](#conf-pure-eval) is disabled, - this is initialised using the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) - environment variable, or, if it is unset and [restricted evaluation](#conf-restrict-eval) - is disabled, a default search path including the user's and `root`'s channels. - )"}; + this, getDefaultNixPath(), "nix-path", + "List of directories to be searched for `<...>` file references."}; Setting<bool> restrictEval{ this, false, "restrict-eval", diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 8061db3df..4beb0b0fe 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -16,7 +16,9 @@ let subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; - flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir); + + flake = import (outPath + "/flake.nix"); inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) @@ -43,7 +45,21 @@ let outputs = flake.outputs (inputs // { self = result; }); - result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; }; + result = + outputs + # We add the sourceInfo attribute for its metadata, as they are + # relevant metadata for the flake. However, the outPath of the + # sourceInfo does not necessarily match the outPath of the flake, + # as the flake may be in a subdirectory of a source. + # This is shadowed in the next // + // sourceInfo + // { + # This shadows the sourceInfo.outPath + inherit outPath; + + inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; + }; + in if node.flake or true then assert builtins.isFunction flake.outputs; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 336eb274d..ac396236f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -125,6 +125,9 @@ static FlakeInput parseFlakeInput(EvalState & state, follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: attrs.emplace(state.symbols[attr.name], attr.value->string.s); @@ -139,6 +142,7 @@ static FlakeInput parseFlakeInput(EvalState & state, throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", state.symbols[attr.name], showType(*attr.value)); } + #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( @@ -320,7 +324,7 @@ LockedFlake lockFlake( const FlakeRef & topRef, const LockFlags & lockFlags) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); FlakeCache flakeCache; @@ -334,10 +338,14 @@ LockedFlake lockFlake( } try { + if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); + } // FIXME: symlink attack auto oldLockFile = LockFile::read( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + lockFlags.referenceLockFilePath.value_or( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); @@ -619,13 +627,20 @@ LockedFlake lockFlake( debug("new lock file: %s", newLockFile); + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto sourcePath = topRef.input.getSourcePath(); + auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; + if (lockFlags.outputLockFilePath) { + outputLockFilePath = lockFlags.outputLockFilePath; + } + /* Check whether we need to / can write the new lock file. */ - if (!(newLockFile == oldLockFile)) { + if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input.getSourcePath()) { + if (outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -633,25 +648,24 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; - - auto path = *sourcePath + "/" + relPath; - - bool lockFileExists = pathExists(path); + bool lockFileExists = pathExists(*outputLockFilePath); if (lockFileExists) { auto s = chomp(diff); if (s.empty()) - warn("updating lock file '%s'", path); + warn("updating lock file '%s'", *outputLockFilePath); else - warn("updating lock file '%s':\n%s", path, s); + warn("updating lock file '%s':\n%s", *outputLockFilePath, s); } else - warn("creating lock file '%s'", path); + warn("creating lock file '%s'", *outputLockFilePath); - newLockFile.write(path); + newLockFile.write(*outputLockFilePath); std::optional<std::string> commitMessage = std::nullopt; if (lockFlags.commitLockFile) { + if (lockFlags.outputLockFilePath) { + throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); + } std::string cm; cm = fetchSettings.commitLockFileSummary.get(); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 10301d8aa..b6f710288 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "flakeref.hh" @@ -117,6 +118,12 @@ struct LockFlags /* Whether to commit changes to flake.lock. */ bool commitLockFile = false; + /* The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ + std::optional<std::string> referenceLockFilePath; + + /* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ + std::optional<Path> outputLockFilePath; + /* Flake inputs to be overridden. */ std::map<InputPath, FlakeRef> inputOverrides; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index c4142fc20..23d19adb1 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 02e9bdfbc..6512509c5 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "flakeref.hh" diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh index e9a2526bd..91439b0aa 100644 --- a/src/libexpr/function-trace.hh +++ b/src/libexpr/function-trace.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index bbd2d3c47..51ef7782a 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" #include "path.hh" diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 84bec4eba..3b8ec000f 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 2171e769b..d243b9cec 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -46,3 +46,5 @@ $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh + +src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index ffe67f97d..4079a7b24 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <map> #include <vector> @@ -186,7 +187,7 @@ struct ExprString : Expr { std::string s; Value v; - ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; + ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -233,7 +234,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; + ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -243,7 +244,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index ffb364a90..97e615c37 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -90,7 +90,7 @@ static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, cons } -static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, +static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos, const nix::EvalState & state) { AttrPath::iterator i; @@ -188,7 +188,7 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals, static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, - std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> & es) + std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es) { if (es.empty()) return new ExprString(""); @@ -268,7 +268,7 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(s2)); + es2->emplace_back(i->first, new ExprString(std::move(s2))); }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); @@ -408,7 +408,7 @@ expr_op | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); } | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector<std::pair<PosIdx, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } @@ -431,14 +431,14 @@ expr_app expr_select : expr_simple '.' attrpath - { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; } | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } - | expr_simple { $$ = $1; } + | expr_simple ; expr_simple @@ -453,9 +453,10 @@ expr_simple | FLOAT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, *$2); + $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2)); + delete $2; } - | path_start PATH_END { $$ = $1; } + | path_start PATH_END | path_start string_parts_interpolated PATH_END { $2->insert($2->begin(), {makeCurPos(@1, data), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); @@ -465,10 +466,10 @@ expr_simple $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__findFile")), {new ExprVar(data->symbols.create("__nixPath")), - new ExprString(path)}); + new ExprString(std::move(path))}); } | URI { - static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); + static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), @@ -533,7 +534,7 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data), data->state); } + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { @@ -542,6 +543,7 @@ binds auto pos = makeCurPos(@3, data); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } + delete $3; } | binds INHERIT '(' expr ')' attrs ';' { $$ = $1; @@ -551,6 +553,7 @@ binds dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } + delete $6; } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; @@ -596,7 +599,7 @@ attrpath ; attr - : ID { $$ = $1; } + : ID | OR_KW { $$ = {"or", 2}; } ; @@ -612,9 +615,9 @@ expr_list formals : formal ',' formals - { $$ = $3; $$->formals.push_back(*$1); } + { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } | formal - { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; } + { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } | { $$ = new ParserFormals; $$->ellipsis = false; } | ELLIPSIS @@ -729,7 +732,7 @@ Expr * EvalState::parseExprFromString(std::string s, const Path & basePath) Expr * EvalState::parseStdin() { - //Activity act(*logger, lvlTalkative, format("parsing standard input")); + //Activity act(*logger, lvlTalkative, "parsing standard input"); auto buffer = drainFD(0); // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); @@ -813,7 +816,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl } else if (hasPrefix(elem.second, "flake:")) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); debug("fetching flake search path element '%s''", elem.second); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; @@ -832,7 +835,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl } } - debug(format("resolved search path element '%s' to '%s'") % elem.second % res.second); + debug("resolved search path element '%s' to '%s'", elem.second, res.second); searchPathResolved[elem.second] = res; return res; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c6f41c4ca..f1bce2fb6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -254,9 +254,16 @@ static RegisterPrimOp primop_import({ .args = {"path"}, // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. If - *path* is a directory, the file ` default.nix ` in that directory - is loaded. Evaluation aborts if the file doesn’t exist or contains + Load, parse and return the Nix expression in the file *path*. + + The value *path* can be a path, a string, or an attribute set with an + `__toString` attribute or a `outPath` attribute (as derivations or flake + inputs typically have). + + If *path* is a directory, the file `default.nix` in that directory + is loaded. + + Evaluation aborts if the file doesn’t exist or contains an incorrect Nix expression. `import` implements Nix’s module system: you can put any Nix expression (such as a set or a function) in a separate file, and use it from Nix expressions in @@ -570,6 +577,9 @@ struct CompareValues return v1->integer < v2->fpoint; if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow<EvalError>(); + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (v1->type()) { case nInt: return v1->integer < v2->integer; @@ -592,6 +602,7 @@ struct CompareValues } default: state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow<EvalError>(); + #pragma GCC diagnostic pop } } catch (Error & e) { if (!errorCtx.empty()) @@ -1141,13 +1152,13 @@ drvName, Bindings * attrs, Value & v) if (i->name == state.sContentAddressed) { contentAddressed = state.forceBool(*i->value, noPos, context_below); if (contentAddressed) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); } else if (i->name == state.sImpure) { isImpure = state.forceBool(*i->value, noPos, context_below); if (isImpure) - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the @@ -3032,9 +3043,9 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. + The return value of each application of `op` is evaluated immediately, + even for intermediate values. )", .fun = prim_foldlStrict, }); @@ -4114,7 +4125,7 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) if (!primOp.experimentalFeature - || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature)) + || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature)) { addPrimOp({ .fun = primOp.fun, diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1cfb4356b..1c5ce219f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "eval.hh" diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 2e924c302..0d0e00fa5 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -4,6 +4,7 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "url.hh" #include <ctime> #include <iomanip> @@ -68,7 +69,16 @@ void emitTreeAttrs( std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; + if (uri.find("://") == std::string::npos) { + const auto p = ParsedURL { + .scheme = defaultScheme, + .authority = "", + .path = uri + }; + return p.to_string(); + } else { + return uri; + } } std::string fixURIForGit(std::string uri, EvalState & state) @@ -180,7 +190,7 @@ static void fetchTree( static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } @@ -343,36 +353,44 @@ static RegisterPrimOp primop_fetchGit({ of the repo at that URL is fetched. Otherwise, it can be an attribute with the following attributes (all except `url` optional): - - url\ - The URL of the repo. + - `url` + + The URL of the repo. + + - `name` (default: *basename of the URL*) + + The name of the directory the repo should be exported to in the store. + + - `rev` (default: *the tip of `ref`*) + + The [Git revision] to fetch. + This is typically a commit hash. + + [Git revision]: https://git-scm.com/docs/git-rev-parse#_specifying_revisions + + - `ref` (default: `HEAD`) + + The [Git reference] under which to look for the requested revision. + This is often a branch or tag name. + + [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References - - name\ - The name of the directory the repo should be exported to in the - store. Defaults to the basename of the URL. + By default, the `ref` value is prefixed with `refs/heads/`. + As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. - - rev\ - The git revision to fetch. Defaults to the tip of `ref`. + - `submodules` (default: `false`) - - ref\ - The git ref to look for the requested revision under. This is - often a branch or tag name. Defaults to `HEAD`. + A Boolean parameter that specifies whether submodules should be checked out. - By default, the `ref` value is prefixed with `refs/heads/`. As - of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts - with `refs/`. + - `shallow` (default: `false`) - - submodules\ - A Boolean parameter that specifies whether submodules should be - checked out. Defaults to `false`. + A Boolean parameter that specifies whether fetching a shallow clone is allowed. - - shallow\ - A Boolean parameter that specifies whether fetching a shallow clone - is allowed. Defaults to `false`. + - `allRefs` - - allRefs\ - Whether to fetch all refs of the repository. With this argument being - true, it's possible to load a `rev` from *any* `ref` (by default only - `rev`s from the specified `ref` are supported). + Whether to fetch all references of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref` + (by default only `rev`s from the specified `ref` are supported). Here are some examples of how to use `fetchGit`. @@ -463,10 +481,10 @@ static RegisterPrimOp primop_fetchGit({ builtins.fetchGit ./work-dir ``` - If the URL points to a local directory, and no `ref` or `rev` is - given, `fetchGit` will use the current content of the checked-out - files, even if they are not committed or added to Git's index. It will - only consider files added to the Git repository, as listed by `git ls-files`. + If the URL points to a local directory, and no `ref` or `rev` is + given, `fetchGit` will use the current content of the checked-out + files, even if they are not committed or added to Git's index. It will + only consider files added to the Git repository, as listed by `git ls-files`. )", .fun = prim_fetchGit, }); diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 288c15602..c97a0a2db 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <list> #include <map> diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 8534d9567..69c932f05 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include <gtest/gtest.h> #include <gmock/gmock.h> diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index e1d3ac503..ce3b5d11f 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -15,8 +15,8 @@ namespace nix { return oss.str(); } - void log(Verbosity lvl, const FormatOrString & fs) override { - oss << fs.s << std::endl; + void log(Verbosity lvl, std::string_view s) override { + oss << s << std::endl; } void logEI(const ErrorInfo & ei) override { diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh index 54d21760e..c0bc97ba3 100644 --- a/src/libexpr/tests/value/context.hh +++ b/src/libexpr/tests/value/context.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 22f26b790..713356c7f 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 3f6222768..341c8922f 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -26,8 +26,8 @@ static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) { if (auto path = std::get_if<Path>(&pos.origin)) xmlAttrs["path"] = *path; - xmlAttrs["line"] = (format("%1%") % pos.line).str(); - xmlAttrs["column"] = (format("%1%") % pos.column).str(); + xmlAttrs["line"] = fmt("%1%", pos.line); + xmlAttrs["column"] = fmt("%1%", pos.column); } @@ -64,7 +64,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, switch (v.type()) { case nInt: - doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); + doc.writeEmptyElement("int", singletonAttrs("value", fmt("%1%", v.integer))); break; case nBool: @@ -156,7 +156,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; case nFloat: - doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + doc.writeEmptyElement("float", singletonAttrs("value", fmt("%1%", v.fpoint))); break; case nThunk: diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 506f32b6b..ace7ead0f 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nixexpr.hh" #include "eval.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 508dbe218..bfae4ee94 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <cassert> diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 721563cba..d467b4f1d 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "comparator.hh" diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index e41037633..1a14bb023 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index 3763ee2a6..ae398d040 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fetchers.hh" diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index f33cbdcfc..6108a179c 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "config.hh" @@ -57,7 +58,7 @@ struct FetchSettings : public Config ``` This example specifies three tokens, one each for accessing - github.com, gitlab.mycompany.com, and sourceforge.net. + github.com, gitlab.mycompany.com, and gitlab.com. The `input.foo` uses the "gitlab" fetcher, which might requires specifying the token type along with the token @@ -75,21 +76,25 @@ struct FetchSettings : public Config Path or URI of the global flake registry. When empty, disables the global flake registry. - )"}; + )", + {}, true, Xp::Flakes}; Setting<bool> useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references."}; + "Whether to use flake registries to resolve flake references.", + {}, true, Xp::Flakes}; Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting."}; + "Whether to accept nix configuration from a flake without prompting.", + {}, true, Xp::Flakes}; Setting<std::string> commitLockFileSummary{ this, "", "commit-lockfile-summary", R"( The commit summary to use when committing changed flake lock files. If empty, the summary is generated based on the action performed. - )"}; + )", + {}, true, Xp::Flakes}; }; // FIXME: don't use a global variable. diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 17da37f47..acdecea57 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -63,6 +64,11 @@ public: one that contains a commit hash or content hash. */ bool isLocked() const { return locked; } + /* Check whether the input carries all necessary info required + for cache insertion and substitution. + These fields are used to uniquely identify cached trees + within the "tarball TTL" window without necessarily + indicating that the input's origin is unchanged. */ bool hasAllInfo() const; bool operator ==(const Input & other) const; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 309a143f5..1da8c9609 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -266,7 +266,7 @@ struct GitInputScheme : InputScheme for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules") + else if (name == "shallow" || name == "submodules" || name == "allRefs") attrs.emplace(name, Explicit<bool> { value == "1" }); else url2.query.emplace(name, value); diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 260a2c460..f57ab1e6b 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "fetchers.hh" diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index f180d83ce..e7ed0d934 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "args.hh" diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index f3c759193..e5721420c 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index e9205a5e5..6600ec177 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -72,6 +72,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool paused = false; bool haveUpdate = true; }; @@ -120,16 +121,28 @@ public: updateThread.join(); } + void pause() override { + state_.lock()->paused = true; + writeToStderr("\r\e[K"); + } + + void resume() override { + state_.lock()->paused = false; + writeToStderr("\r\e[K"); + state_.lock()->haveUpdate = true; + updateCV.notify_one(); + } + bool isVerbose() override { return printBuildLogs; } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; auto state(state_.lock()); - log(*state, lvl, fs.s); + log(*state, lvl, s); } void logEI(const ErrorInfo & ei) override @@ -142,7 +155,7 @@ public: log(*state, ei.level, oss.str()); } - void log(State & state, Verbosity lvl, const std::string & s) + void log(State & state, Verbosity lvl, std::string_view s) { if (state.active) { writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); @@ -339,7 +352,7 @@ public: auto nextWakeup = std::chrono::milliseconds::max(); state.haveUpdate = false; - if (!state.active) return nextWakeup; + if (state.paused || !state.active) return nextWakeup; std::string line; diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 3a76f8448..c3c6e3833 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d4871a8e2..37664c065 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -84,8 +84,18 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild, downloadSizeMiB, narSizeMiB); } - for (auto & i : willSubstitute) - printMsg(lvl, " %s", store->printStorePath(i)); + std::vector<const StorePath *> willSubstituteSorted = {}; + std::for_each(willSubstitute.begin(), willSubstitute.end(), + [&](const StorePath &p) { willSubstituteSorted.push_back(&p); }); + std::sort(willSubstituteSorted.begin(), willSubstituteSorted.end(), + [](const StorePath *lhs, const StorePath *rhs) { + if (lhs->name() == rhs->name()) + return lhs->to_string() < rhs->to_string(); + else + return lhs->name() < rhs->name(); + }); + for (auto p : willSubstituteSorted) + printMsg(lvl, " %s", store->printStorePath(*p)); } if (!unknown.empty()) { @@ -347,7 +357,7 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName) { - std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; + std::cout << fmt("%1% (Nix) %2%", programName, nixVersion) << std::endl; if (verbosity > lvlInfo) { Strings cfg; #if HAVE_BOEHMGC diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 1715374a6..d915a4a65 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "args.hh" diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index abd92a83c..5e52d7844 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "store-api.hh" @@ -16,17 +17,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; - const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + + const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + "Whether to write a JSON file that lists the files in each NAR."}; + + const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )"}; + + const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", + "Path to the secret key used to sign the binary cache."}; + + const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression for NARs, available for xz and zstd only currently"}; + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level", - "specify 'preset level' of compression to be used with NARs: " - "meaning and accepted range of values depends on compression method selected, " - "other than -1 which we reserve to indicate Nix defaults should be used"}; + R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )"}; }; class BinaryCacheStore : public virtual BinaryCacheStoreConfig, diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a5749cf33..a12c599d9 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "realisation.hh" #include "derived-path.hh" @@ -52,6 +53,7 @@ struct BuildResult case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2021d0023..26faf8c8e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -199,10 +199,10 @@ void DerivationGoal::haveDerivation() parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); if (!drv->type().hasKnownOutputPaths()) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (!drv->type().isPure()) { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); for (auto & [outputName, output] : drv->outputs) { auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); @@ -336,7 +336,7 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(i.first); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", @@ -477,7 +477,7 @@ void DerivationGoal::inputsRealised() ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ - ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) /* Must resolve if floating and there are any inputs drvs. */ : true); @@ -488,7 +488,7 @@ void DerivationGoal::inputsRealised() }, drvType.raw()); if (resolveDrv && !fullDrv.inputDrvs.empty()) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a @@ -732,7 +732,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random()); if (pathExists(storePath)) movePath(storePath, oldPath); @@ -911,7 +911,11 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) + ? "nix log" + : "nix-store -l"; + msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + nixLogCommand, worker.store.printStorePath(drvPath)); } @@ -1352,7 +1356,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() }; } auto drvOutput = DrvOutput{info.outputHash, i.first}; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 707e38b4b..f43ce22af 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "parsed-derivations.hh" #include "lock.hh" diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index b7f7b5ab1..b30957c84 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -61,20 +61,25 @@ void DrvOutputSubstitutionGoal::tryNext() // FIXME: Make async // outputInfo = sub->queryRealisation(id); - outPipe.create(); - promise = decltype(promise)(); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + downloadState = std::make_shared<DownloadState>(); + downloadState->outPipe.create(); sub->queryRealisation( - id, { [&](std::future<std::shared_ptr<const Realisation>> res) { + id, + { [downloadState(downloadState)](std::future<std::shared_ptr<const Realisation>> res) { try { - Finally updateStats([this]() { outPipe.writeSide.close(); }); - promise.set_value(res.get()); + Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); + downloadState->promise.set_value(res.get()); } catch (...) { - promise.set_exception(std::current_exception()); + downloadState->promise.set_exception(std::current_exception()); } } }); - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false); state = &DrvOutputSubstitutionGoal::realisationFetched; } @@ -84,7 +89,7 @@ void DrvOutputSubstitutionGoal::realisationFetched() worker.childTerminated(this); try { - outputInfo = promise.get_future().get(); + outputInfo = downloadState->promise.get_future().get(); } catch (std::exception & e) { printError(e.what()); substituterFailed = true; @@ -155,7 +160,7 @@ void DrvOutputSubstitutionGoal::work() void DrvOutputSubstitutionGoal::handleEOF(int fd) { - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 948dbda8f..3b6620b76 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "goal.hh" @@ -16,7 +17,7 @@ class Worker; // 2. Substitute the corresponding output path // 3. Register the output info class DrvOutputSubstitutionGoal : public Goal { -private: + // The drv output we're trying to substitue DrvOutput id; @@ -30,9 +31,13 @@ private: /* The current substituter. */ std::shared_ptr<Store> sub; - Pipe outPipe; - std::thread thr; - std::promise<std::shared_ptr<const Realisation>> promise; + struct DownloadState + { + Pipe outPipe; + std::promise<std::shared_ptr<const Realisation>> promise; + }; + + std::shared_ptr<DownloadState> downloadState; /* Whether a substituter failed. */ bool substituterFailed = false; diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 58e805f55..d59b94797 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -78,9 +78,9 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex) } -void Goal::trace(const FormatOrString & fs) +void Goal::trace(std::string_view s) { - debug("%1%: %2%", name, fs.s); + debug("%1%: %2%", name, s); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 35121c5d9..924a8bbd5 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "store-api.hh" @@ -88,7 +89,7 @@ struct Goal : public std::enable_shared_from_this<Goal> abort(); } - void trace(const FormatOrString & fs); + void trace(std::string_view s); std::string getName() { diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index cb58a1f02..075ad554f 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -35,7 +35,10 @@ HookInstance::HookInstance() /* Fork the hook. */ pid = startProcess([&]() { - commonChildInit(fromHook); + if (dup2(fromHook.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + commonChildInit(); if (chdir("/") == -1) throw SysError("changing into /"); diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 9e8cff128..6bf60b297 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" #include "serialise.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7b125f5d2..e22180670 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -292,7 +292,7 @@ void LocalDerivationGoal::closeReadPipes() if (hook) { DerivationGoal::closeReadPipes(); } else - builderOut.readSide = -1; + builderOut.close(); } @@ -413,7 +413,7 @@ void LocalDerivationGoal::startBuilder() ) { #if __linux__ - settings.requireExperimentalFeature(Xp::Cgroups); + experimentalFeatureSettings.require(Xp::Cgroups); auto cgroupFS = getCgroupFS(); if (!cgroupFS) @@ -650,7 +650,7 @@ void LocalDerivationGoal::startBuilder() /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir); // FIXME: make this 0700 if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) @@ -753,8 +753,7 @@ void LocalDerivationGoal::startBuilder() throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); + printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { @@ -803,15 +802,13 @@ void LocalDerivationGoal::startBuilder() /* Create the log file. */ Path logFile = openLogFile(); - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) + /* Create a pseudoterminal to get the output of the builder. */ + builderOut = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut) throw SysError("opening pseudoterminal master"); // FIXME: not thread-safe, use ptsname_r - std::string slaveName(ptsname(builderOut.readSide.get())); + std::string slaveName = ptsname(builderOut.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -822,34 +819,34 @@ void LocalDerivationGoal::startBuilder() } #if __APPLE__ else { - if (grantpt(builderOut.readSide.get())) + if (grantpt(builderOut.get())) throw SysError("granting access to pseudoterminal slave"); } #endif - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - - if (unlockpt(builderOut.readSide.get())) + if (unlockpt(builderOut.get())) throw SysError("unlocking pseudoterminal"); - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); + /* Open the slave side of the pseudoterminal and use it as stderr. */ + auto openSlave = [&]() + { + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.get(), &term)) + throw SysError("getting pseudoterminal attributes"); - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); + cfmakeraw(&term); - cfmakeraw(&term); + if (tcsetattr(builderOut.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); + if (dup2(builderOut.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + }; buildResult.startTime = time(0); @@ -898,7 +895,16 @@ void LocalDerivationGoal::startBuilder() usingUserNamespace = userNamespacesSupported(); + Pipe sendPid; + sendPid.create(); + Pid helper = startProcess([&]() { + sendPid.readSide.close(); + + /* We need to open the slave early, before + CLONE_NEWUSER. Otherwise we get EPERM when running as + root. */ + openSlave(); /* Drop additional groups here because we can't do it after we've created the new user namespace. FIXME: @@ -920,11 +926,12 @@ void LocalDerivationGoal::startBuilder() pid_t child = startProcess([&]() { runChild(); }, options); - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); }); + sendPid.writeSide.close(); + if (helper.wait() != 0) throw Error("unable to start build process"); @@ -936,10 +943,9 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - pid = string2Int<pid_t>(ss[1]).value(); + auto ss = tokenizeString<std::vector<std::string>>(readLine(sendPid.readSide.get())); + assert(ss.size() == 1); + pid = string2Int<pid_t>(ss[0]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -971,10 +977,6 @@ void LocalDerivationGoal::startBuilder() "nobody:x:65534:65534:Nobody:/:/noshell\n", sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - /* Make /etc unwritable */ - if (!parsedDrv->useUidRange()) - chmod_(chrootRootDir + "/etc", 0555); - /* Save the mount- and user namespace of the child. We have to do this *before* the child does a chroot. */ sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); @@ -998,21 +1000,21 @@ void LocalDerivationGoal::startBuilder() #endif { pid = startProcess([&]() { + openSlave(); runChild(); }); } /* parent */ pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); /* Check if setting up the build environment failed. */ std::vector<std::string> msgs; while (true) { std::string msg = [&]() { try { - return readLine(builderOut.readSide.get()); + return readLine(builderOut.get()); } catch (Error & e) { auto status = pid.wait(); e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", @@ -1024,7 +1026,7 @@ void LocalDerivationGoal::startBuilder() }(); if (msg.substr(0, 1) == "\2") break; if (msg.substr(0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); + FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); throw ex; @@ -1108,7 +1110,7 @@ void LocalDerivationGoal::initEnv() env["NIX_STORE"] = worker.store.storeDir; /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); initTmpDir(); @@ -1159,10 +1161,10 @@ void LocalDerivationGoal::writeStructuredAttrs() writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); - env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); chownToBuilder(tmpDir + "/.attrs.json"); - env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; + env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; } } @@ -1418,7 +1420,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature(Xp::RecursiveNix); + experimentalFeatureSettings.require(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; @@ -1654,7 +1656,7 @@ void LocalDerivationGoal::runChild() try { /* child */ - commonChildInit(builderOut); + commonChildInit(); try { setupSeccomp(); @@ -1855,6 +1857,10 @@ void LocalDerivationGoal::runChild() } } + /* Make /etc unwritable */ + if (!parsedDrv->useUidRange()) + chmod_(chrootRootDir + "/etc", 0555); + /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount namespace. This means that the call to setns() in @@ -2063,7 +2069,7 @@ void LocalDerivationGoal::runChild() /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true); /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); @@ -2274,7 +2280,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); + experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); @@ -2694,7 +2700,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && drv->type().isPure()) { signRealisation(thisRealisation); @@ -2892,7 +2898,7 @@ void LocalDerivationGoal::deleteTmpDir(bool force) bool LocalDerivationGoal::isReadDesc(int fd) { return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builderOut.readSide.get()); + (!hook && fd == builderOut.get()); } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 34c4e9187..1c4b4e3fe 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivation-goal.hh" #include "local-store.hh" @@ -24,8 +25,9 @@ struct LocalDerivationGoal : public DerivationGoal /* The path of the temporary directory in the sandbox. */ Path tmpDirInSandbox; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; + /* Master side of the pseudoterminal used for the builder's + standard output/error. */ + AutoCloseFD builderOut; /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh index 30e4f4062..91b730fab 100644 --- a/src/libstore/build/personality.hh +++ b/src/libstore/build/personality.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string> diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index a73f8e666..1add9eb14 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "lock.hh" #include "store-api.hh" diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6d68d3cf1..d840b3b3f 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "lock.hh" diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 66597e456..d201fb3ac 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index b1fbda13d..7bba33fb9 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -92,13 +92,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (S_ISLNK(dstSt.st_mode)) { auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) - throw Error( - "files '%1%' and '%2%' have the same priority %3%; " - "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or type 'nix profile install --help' if using 'nix profile' to find out how " - "to change the priority of one of the conflicting packages" - " (0 being the highest priority)", - srcFile, readLink(dstFile), priority); + throw BuildEnvFileConflictError( + readLink(dstFile), + srcFile, + priority + ); if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 73c0f5f7f..0923c2adb 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" @@ -12,6 +13,32 @@ struct Package { Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; +class BuildEnvFileConflictError : public Error +{ +public: + const Path fileA; + const Path fileB; + int priority; + + BuildEnvFileConflictError( + const Path fileA, + const Path fileB, + int priority + ) + : Error( + "Unable to build profile. There is a conflict for the following files:\n" + "\n" + " %1%\n" + " %2%", + fileA, + fileB + ) + , fileA(fileA) + , fileB(fileB) + , priority(priority) + {} +}; + typedef std::vector<Package> Packages; void buildProfile(const Path & out, Packages && pkgs); diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..19fdfc1eb 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -1,48 +1,80 @@ #pragma once +///@file #include <variant> #include "hash.hh" namespace nix { +/** + * An enumeration of the ways we can serialize file system objects. + */ enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ Flat = false, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in Nix + * Archive format and ingest that + */ Recursive = true }; +/** + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + */ struct TextHash { + /** + * Hash of the contents of the text/file. + */ Hash hash; }; -/// Pair of a hash, and how the file system was ingested +/** + * For path computed by makeFixedOutputPath. + */ struct FixedOutputHash { + /** + * How the file system objects are serialized + */ FileIngestionMethod method; + /** + * Hash of that serialization + */ Hash hash; + std::string printMethodAlgo() const; }; -/* - We've accumulated several types of content-addressed paths over the years; - fixed-output derivations support multiple hash algorithms and serialisation - methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:<sha256 hash of file contents>’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:<r?>:<ht>:<h>’: For paths computed by - makeFixedOutputPath() / addToStore(). -*/ +/** + * We've accumulated several types of content-addressed paths over the + * years; fixed-output derivations support multiple hash algorithms and + * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the + * following forms: + * + * - ‘text:sha256:<sha256 hash of file contents>’: For paths + * computed by Store::makeTextPath() / Store::addTextToStore(). + * + * - ‘fixed:<r?>:<ht>:<h>’: For paths computed by + * Store::makeFixedOutputPath() / Store::addToStore(). + */ typedef std::variant< - TextHash, // for paths computed by makeTextPath() / addTextToStore - FixedOutputHash // for path computed by makeFixedOutputPath + TextHash, + FixedOutputHash > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ +/** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ +/** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); std::string renderContentAddress(ContentAddress ca); @@ -65,6 +97,11 @@ struct FixedOutputHashMethod { HashType hashType; }; +/** + * Ways of content addressing but not a complete ContentAddress. + * + * A ContentAddress without a Hash. + */ typedef std::variant< TextHashMethod, FixedOutputHashMethod diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 03f85c103..a98f2a3b8 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5e6fd011f..656ad4587 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -67,12 +67,12 @@ struct TunnelLogger : public Logger state->pendingMsgs.push_back(s); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); + buf << STDERR_NEXT << (s + "\n"); enqueueMsg(buf.s); } @@ -231,10 +231,10 @@ struct ClientSettings try { if (name == "ssh-auth-sock") // obsolete ; - else if (name == settings.experimentalFeatures.name) { + else if (name == experimentalFeatureSettings.experimentalFeatures.name) { // We don’t want to forward the experimental features to // the daemon, as that could cause some pretty weird stuff - if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get()) + if (parseFeatures(tokenizeString<StringSet>(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); } else if (name == settings.pluginFiles.name) { if (tokenizeString<Paths>(value) != settings.pluginFiles.get()) diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 8c765615c..67340a05b 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "serialise.hh" #include "store-api.hh" diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index cf18724ef..06cc69056 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -5,6 +5,7 @@ #include "worker-protocol.hh" #include "fs-accessor.hh" #include <boost/container/small_vector.hpp> +#include <nlohmann/json.hpp> namespace nix { @@ -220,7 +221,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, } const auto hashType = parseHashType(hashAlgo); if (hash == "impure") { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); assert(pathS == ""); return DerivationOutput::Impure { .method = std::move(method), @@ -235,7 +236,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); assert(pathS == ""); return DerivationOutput::CAFloating { .method = std::move(method), @@ -890,4 +891,63 @@ std::optional<BasicDerivation> Derivation::tryResolve( const Hash impureOutputHash = hashString(htSHA256, "impure"); +nlohmann::json DerivationOutput::toJSON( + const Store & store, std::string_view drvName, std::string_view outputName) const +{ + nlohmann::json res = nlohmann::json::object(); + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doi) { + res["path"] = store.printStorePath(doi.path); + }, + [&](const DerivationOutput::CAFixed & dof) { + res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + res["hashAlgo"] = dof.hash.printMethodAlgo(); + res["hash"] = dof.hash.hash.to_string(Base16, false); + }, + [&](const DerivationOutput::CAFloating & dof) { + res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); + }, + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); + res["impure"] = true; + }, + }, raw()); + return res; +} + +nlohmann::json Derivation::toJSON(const Store & store) const +{ + nlohmann::json res = nlohmann::json::object(); + + { + nlohmann::json & outputsObj = res["outputs"]; + outputsObj = nlohmann::json::object(); + for (auto & [outputName, output] : outputs) { + outputsObj[outputName] = output.toJSON(store, name, outputName); + } + } + + { + auto& inputsList = res["inputSrcs"]; + inputsList = nlohmann::json ::array(); + for (auto & input : inputSrcs) + inputsList.emplace_back(store.printStorePath(input)); + } + + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json ::object(); + for (auto & input : inputDrvs) + inputDrvsObj[store.printStorePath(input.first)] = input.second; + } + + res["system"] = platform; + res["builder"] = builder; + res["args"] = args; + res["env"] = env; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f42c13cdc..e12bd2119 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "path.hh" #include "types.hh" @@ -17,42 +18,72 @@ class Store; /* Abstract syntax of derivations. */ -/* The traditional non-fixed-output derivation type. */ +/** + * The traditional non-fixed-output derivation type. + */ struct DerivationOutputInputAddressed { StorePath path; }; -/* Fixed-output derivations, whose output paths are content addressed - according to that fixed output. */ +/** + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. + */ struct DerivationOutputCAFixed { - FixedOutputHash hash; /* hash used for expected hash computation */ + /** + * hash used for expected hash computation + */ + FixedOutputHash hash; + + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; }; -/* Floating-output derivations, whose output paths are content addressed, but - not fixed, and so are dynamically calculated from whatever the output ends - up being. */ +/** + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ struct DerivationOutputCAFloating { - /* information used for expected hash computation */ + /** + * How the file system objects will be serialized for hashing + */ FileIngestionMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; }; -/* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known yet. +/** + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. */ struct DerivationOutputDeferred {}; -/* Impure output which is moved to a content-addressed location (like - CAFloating) but isn't registered as a realization. +/** + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. */ struct DerivationOutputImpure { - /* information used for expected hash computation */ + /** + * How the file system objects will be serialized for hashing + */ FileIngestionMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; }; @@ -64,6 +95,9 @@ typedef std::variant< DerivationOutputImpure > _DerivationOutputRaw; +/** + * A single output of a BasicDerivation (and Derivation). + */ struct DerivationOutput : _DerivationOutputRaw { using Raw = _DerivationOutputRaw; @@ -75,38 +109,81 @@ struct DerivationOutput : _DerivationOutputRaw using Deferred = DerivationOutputDeferred; using Impure = DerivationOutputImpure; - /* Note, when you use this function you should make sure that you're passing - the right derivation name. When in doubt, you should use the safer - interface provided by BasicDerivation::outputsAndOptPaths */ + /** + * \note when you use this function you should make sure that you're + * passing the right derivation name. When in doubt, you should use + * the safer interface provided by + * BasicDerivation::outputsAndOptPaths + */ std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const; inline const Raw & raw() const { return static_cast<const Raw &>(*this); } + + nlohmann::json toJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName) const; }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; -/* These are analogues to the previous DerivationOutputs data type, but they - also contains, for each output, the (optional) store path in which it would - be written. To calculate values of these types, see the corresponding - functions in BasicDerivation */ +/** + * These are analogues to the previous DerivationOutputs data type, + * but they also contains, for each output, the (optional) store + * path in which it would be written. To calculate values of these + * types, see the corresponding functions in BasicDerivation. + */ typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePath>>> DerivationOutputsAndOptPaths; -/* For inputs that are sub-derivations, we specify exactly which - output IDs we are interested in. */ +/** + * For inputs that are sub-derivations, we specify exactly which + * output IDs we are interested in. + */ typedef std::map<StorePath, StringSet> DerivationInputs; +/** + * Input-addressed derivation types + */ struct DerivationType_InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ bool deferred; }; +/** + * Content-addressed derivation types + */ struct DerivationType_ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating. + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ bool fixed; }; +/** + * Impure derivation type + * + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. + */ struct DerivationType_Impure { }; @@ -123,30 +200,38 @@ struct DerivationType : _DerivationTypeRaw { using ContentAddressed = DerivationType_ContentAddressed; using Impure = DerivationType_Impure; - /* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ + /** + * Do the outputs of the derivation have paths calculated from their + * content, or from the derivation itself? + */ bool isCA() const; - /* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ + /** + * Is the content of the outputs fixed <em>a priori</em> via a hash? + * Never true for non-CA derivations. + */ bool isFixed() const; - /* Whether the derivation is fully sandboxed. If false, the - sandbox is opened up, e.g. the derivation has access to the - network. Note that whether or not we actually sandbox the - derivation is controlled separately. Always true for non-CA - derivations. */ + /** + * Whether the derivation is fully sandboxed. If false, the sandbox + * is opened up, e.g. the derivation has access to the network. Note + * that whether or not we actually sandbox the derivation is + * controlled separately. Always true for non-CA derivations. + */ bool isSandboxed() const; - /* Whether the derivation is expected to produce the same result - every time, and therefore it only needs to be built once. This - is only false for derivations that have the attribute '__impure - = true'. */ + /** + * Whether the derivation is expected to produce the same result + * every time, and therefore it only needs to be built once. This is + * only false for derivations that have the attribute '__impure = + * true'. + */ bool isPure() const; - /* Does the derivation knows its own output paths? - Only true when there's no floating-ca derivation involved in the - closure, or if fixed output. + /** + * Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the + * closure, or if fixed output. */ bool hasKnownOutputPaths() const; @@ -170,15 +255,21 @@ struct BasicDerivation bool isBuiltin() const; - /* Return true iff this is a fixed-output derivation. */ + /** + * Return true iff this is a fixed-output derivation. + */ DerivationType type() const; - /* Return the output names of a derivation. */ + /** + * Return the output names of a derivation. + */ StringSet outputNames() const; - /* Calculates the maps that contains all the DerivationOutputs, but - augmented with knowledge of the Store paths they would be written - into. */ + /** + * Calculates the maps that contains all the DerivationOutputs, but + * augmented with knowledge of the Store paths they would be written + * into. + */ DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); @@ -186,23 +277,33 @@ struct BasicDerivation struct Derivation : BasicDerivation { - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + /** + * inputs that are sub-derivations + */ + DerivationInputs inputDrvs; - /* Print a derivation. */ + /** + * Print a derivation. + */ std::string unparse(const Store & store, bool maskOutputs, std::map<std::string, StringSet> * actualInputs = nullptr) const; - /* Return the underlying basic derivation but with these changes: - - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. - - 2. Input placeholders are replaced with realized input store paths. */ + /** + * Return the underlying basic derivation but with these changes: + * + * 1. Input drvs are emptied, but the outputs of them that were used + * are added directly to input sources. + * + * 2. Input placeholders are replaced with realized input store + * paths. + */ std::optional<BasicDerivation> tryResolve(Store & store) const; - /* Like the above, but instead of querying the Nix database for - realisations, uses a given mapping from input derivation paths - + output names to actual output store paths. */ + /** + * Like the above, but instead of querying the Nix database for + * realisations, uses a given mapping from input derivation paths + + * output names to actual output store paths. + */ std::optional<BasicDerivation> tryResolve( Store & store, const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; @@ -210,86 +311,115 @@ struct Derivation : BasicDerivation Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + + nlohmann::json toJSON(const Store & store) const; }; class Store; -/* Write a derivation to the Nix store, and return its path. */ +/** + * Write a derivation to the Nix store, and return its path. + */ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, bool readOnly = false); -/* Read a derivation from a file. */ +/** + * Read a derivation from a file. + */ Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); -// FIXME: remove +/** + * \todo Remove. + * + * Use Path::isDerivation instead. + */ bool isDerivation(std::string_view fileName); -/* Calculate the name that will be used for the store path for this - output. - - This is usually <drv-name>-<output-name>, but is just <drv-name> when - the output name is "out". */ +/** + * Calculate the name that will be used for the store path for this + * output. + * + * This is usually <drv-name>-<output-name>, but is just <drv-name> when + * the output name is "out". + */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// The hashes modulo of a derivation. -// -// Each output is given a hash, although in practice only the content-addressed -// derivations (fixed-output or not) will have a different hash for each -// output. +/** + * The hashes modulo of a derivation. + * + * Each output is given a hash, although in practice only the content-addressed + * derivations (fixed-output or not) will have a different hash for each + * output. + */ struct DrvHash { + /** + * Map from output names to hashes + */ std::map<std::string, Hash> hashes; enum struct Kind : bool { - // Statically determined derivations. - // This hash will be directly used to compute the output paths + /** + * Statically determined derivations. + * This hash will be directly used to compute the output paths + */ Regular, - // Floating-output derivations (and their reverse dependencies). + + /** + * Floating-output derivations (and their reverse dependencies). + */ Deferred, }; + /** + * The kind of derivation this is, simplified for just "derivation hash + * modulo" purposes. + */ Kind kind; }; void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -/* Returns hashes with the details of fixed-output subderivations - expunged. - - A fixed-output derivation is a derivation whose outputs have a - specified content hash and hash algorithm. (Currently they must have - exactly one output (`out'), which is specified using the `outputHash' - and `outputHashAlgo' attributes, but the algorithm doesn't assume - this.) We don't want changes to such derivations to propagate upwards - through the dependency graph, changing output paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it---after - all, (the hash of) the file being downloaded is unchanged. So the - *output paths* should not change. On the other hand, the *derivation - paths* should change to reflect the new dependency graph. - - For fixed-output derivations, this returns a map from the name of - each output to its hash, unique up to the output's contents. - - For regular derivations, it returns a single hash of the derivation - ATerm, after subderivations have been likewise expunged from that - derivation. +/** + * Returns hashes with the details of fixed-output subderivations + * expunged. + * + * A fixed-output derivation is a derivation whose outputs have a + * specified content hash and hash algorithm. (Currently they must have + * exactly one output (`out'), which is specified using the `outputHash' + * and `outputHashAlgo' attributes, but the algorithm doesn't assume + * this.) We don't want changes to such derivations to propagate upwards + * through the dependency graph, changing output paths everywhere. + * + * For instance, if we change the url in a call to the `fetchurl' + * function, we do not want to rebuild everything depending on it---after + * all, (the hash of) the file being downloaded is unchanged. So the + * *output paths* should not change. On the other hand, the *derivation + * paths* should change to reflect the new dependency graph. + * + * For fixed-output derivations, this returns a map from the name of + * each output to its hash, unique up to the output's contents. + * + * For regular derivations, it returns a single hash of the derivation + * ATerm, after subderivations have been likewise expunged from that + * derivation. */ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); -/* - Return a map associating each output to a hash that uniquely identifies its - derivation (modulo the self-references). - - FIXME: what is the Hash in this map? +/** + * Return a map associating each output to a hash that uniquely identifies its + * derivation (modulo the self-references). + * + * \todo What is the Hash in this map? */ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv); -/* Memoisation of hashDerivationModulo(). */ +/** + * Memoisation of hashDerivationModulo(). + */ typedef std::map<StorePath, DrvHash> DrvHashes; // FIXME: global, though at least thread-safe. @@ -301,21 +431,25 @@ struct Sink; Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); -/* This creates an opaque and almost certainly unique string - deterministically from the output name. - - It is used as a placeholder to allow derivations to refer to their - own outputs without needing to use the hash of a derivation in - itself, making the hash near-impossible to calculate. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from the output name. + * + * It is used as a placeholder to allow derivations to refer to their + * own outputs without needing to use the hash of a derivation in + * itself, making the hash near-impossible to calculate. + */ std::string hashPlaceholder(const std::string_view outputName); -/* This creates an opaque and almost certainly unique string - deterministically from a derivation path and output name. - - It is used as a placeholder to allow derivations to refer to - content-addressed paths whose content --- and thus the path - themselves --- isn't yet known. This occurs when a derivation has a - dependency which is a CA derivation. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from a derivation path and output name. + * + * It is used as a placeholder to allow derivations to refer to + * content-addressed paths whose content --- and thus the path + * themselves --- isn't yet known. This occurs when a derivation has a + * dependency which is a CA derivation. + */ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e0d86a42f..e5f0f1b33 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -105,7 +105,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath)); for (auto& [outputName, outputPath] : p.outputs) { - if (settings.isExperimentalFeatureEnabled( + if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { auto drvOutput = get(drvHashes, outputName); if (!drvOutput) diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9e0cce377..2155776b1 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "path.hh" @@ -105,7 +106,7 @@ using _BuiltPathRaw = std::variant< >; /** - * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * A built path. Similar to a DerivedPath, but enriched with the corresponding * output path(s). */ struct BuiltPath : _BuiltPathRaw { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b4fbe0b70..16e5fafd7 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; const std::string name() override { return "Dummy Store"; } + + std::string doc() override + { + return + #include "dummy-store.md" + ; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store diff --git a/src/libstore/dummy-store.md b/src/libstore/dummy-store.md new file mode 100644 index 000000000..eb7b4ba0d --- /dev/null +++ b/src/libstore/dummy-store.md @@ -0,0 +1,13 @@ +R"( + +**Store URL format**: `dummy://` + +This store type represents a store that contains no store paths and +cannot be written to. It's useful when you want to use the Nix +evaluator when no actual Nix store exists, e.g. + +```console +# nix eval --store dummy:// --expr '1 + 2' +``` + +)" diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9875da909..4eb838b68 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -16,7 +16,7 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); //logger->incProgress(doneLabel); @@ -71,7 +71,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto path = parseStorePath(readString(source)); - //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {}); auto deriver = readString(source); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b25089ec3..2346accbe 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -88,6 +88,10 @@ struct curlFileTransfer : public FileTransfer {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { + if (errorSink) { + (*errorSink)(data); + } + if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); @@ -163,8 +167,6 @@ struct curlFileTransfer : public FileTransfer } } - if (errorSink) - (*errorSink)({(char *) contents, realSize}); (*decompressionSink)({(char *) contents, realSize}); return realSize; @@ -183,7 +185,7 @@ struct curlFileTransfer : public FileTransfer { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); std::smatch match; if (std::regex_match(line, match, statusLine)) { @@ -207,7 +209,7 @@ struct curlFileTransfer : public FileTransfer long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); if (result.etag == request.expectedETag && httpStatus == 200) { - debug(format("shutting down on 200 HTTP response with expected ETag")); + debug("shutting down on 200 HTTP response with expected ETag"); return 0; } } else if (name == "content-encoding") @@ -316,7 +318,7 @@ struct curlFileTransfer : public FileTransfer if (request.verifyTLS) { if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); } else { curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); @@ -405,6 +407,10 @@ struct curlFileTransfer : public FileTransfer err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -425,6 +431,7 @@ struct curlFileTransfer : public FileTransfer default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; @@ -829,7 +836,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) { auto state(_state->lock()); - while (state->data.empty()) { + if (state->data.empty()) { if (state->quit) { if (state->exc) std::rethrow_exception(state->exc); @@ -837,6 +844,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } state.wait(state->avail); + + if (state->data.empty()) continue; } chunk = std::move(state->data); diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 07d58f53a..378c6ff78 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -87,39 +88,56 @@ struct FileTransfer { virtual ~FileTransfer() { } - /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a FileTransferError - exception. */ + /** + * Enqueue a data transfer request, returning a future to the result of + * the download. The future may throw a FileTransferError + * exception. + */ virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0; std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request); - /* Synchronously download a file. */ + /** + * Synchronously download a file. + */ FileTransferResult download(const FileTransferRequest & request); - /* Synchronously upload a file. */ + /** + * Synchronously upload a file. + */ FileTransferResult upload(const FileTransferRequest & request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + /** + * Download a file, writing its data to a sink. The sink will be + * invoked on the thread of the caller. + */ void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared FileTransfer object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ +/** + * @return a shared FileTransfer object. + * + * Using this object is preferred because it enables connection reuse + * and HTTP/2 multiplexing. + */ ref<FileTransfer> getFileTransfer(); -/* Return a new FileTransfer object. */ +/** + * @return a new FileTransfer object + * + * Prefer getFileTransfer() to this; see its docs for why. + */ ref<FileTransfer> makeFileTransfer(); class FileTransferError : public Error { public: FileTransfer::Error error; - std::optional<std::string> response; // intentionally optional + /// intentionally optional + std::optional<std::string> response; template<typename... Args> FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args); diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index c825e84f2..1c98a42d7 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index b3cbbad74..17f043a63 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 996f26a95..0038ec802 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -34,8 +34,7 @@ static void makeSymlink(const Path & link, const Path & target) createDirs(dirOf(link)); /* Create the new symlink. */ - Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % random()).str(); + Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), random()); createSymlink(target, tempLink); /* Atomically replace the old one. */ @@ -197,7 +196,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) pid_t pid = std::stoi(i.name); - debug(format("reading temporary root file '%1%'") % path); + debug("reading temporary root file '%1%'", path); AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); if (!fd) { /* It's okay if the file has disappeared. */ @@ -263,7 +262,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) target = absPath(target, dirOf(path)); if (!pathExists(target)) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { - printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); + printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } } else { @@ -372,29 +371,29 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError("opening %1%", fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError("iterating /proc/%1%/fd", ent->d_name); - } - fdDir.reset(); - try { + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError("opening %1%", fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); + } + fdDir.reset(); + auto mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { @@ -863,7 +862,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) continue; } - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + printMsg(lvlTalkative, "deleting unused link '%1%'", path); if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8e33a3dec..823b4af74 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -30,28 +30,23 @@ static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixStore(canonPath(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) + , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) + , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; - caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (caFile == "") { - for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) { - caFile = fn; - break; - } - } + auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (sslOverride != "") + caFile = sslOverride; /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); @@ -166,18 +161,6 @@ StringSet Settings::getDefaultExtraPlatforms() return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) -{ - auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), feature) != f.end(); -} - -void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) -{ - if (!isExperimentalFeatureEnabled(feature)) - throw MissingExperimentalFeature(feature); -} - bool Settings::isWSL1() { struct utsname utsbuf; @@ -187,6 +170,13 @@ bool Settings::isWSL1() return hasSuffix(utsbuf.release, "-Microsoft"); } +Path Settings::getDefaultSSLCertFile() +{ + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathExists(fn)) return fn; + return ""; +} + const std::string nixVersion = PACKAGE_VERSION; NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0a4912f67..9b04c9e78 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,9 +1,9 @@ #pragma once +///@file #include "types.hh" #include "config.hh" #include "util.hh" -#include "experimental-features.hh" #include <map> #include <limits> @@ -64,6 +64,8 @@ class Settings : public Config { bool isWSL1(); + Path getDefaultSSLCertFile(); + public: Settings(); @@ -97,7 +99,12 @@ public: Path nixDaemonSocketFile; Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", - "The default Nix store to use."}; + R"( + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for most operations. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) + for supported store types and settings. + )"}; Setting<bool> keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; @@ -201,7 +208,16 @@ public: {"build-timeout"}}; PathSetting buildHook{this, true, "", "build-hook", - "The path of the helper program that executes builds to remote machines."}; + R"( + The path to the helper program that executes remote builds. + + Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. + The default value is the internal Nix binary that implements remote building. + + > **Important** + > + > Change this setting only if you really know what you’re doing. + )"}; Setting<std::string> builders{ this, "@" + nixConfDir + "/machines", "builders", @@ -669,8 +685,9 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. Substituters - are tried based on their Priority value, which each substituter can set + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used as substituters, separated by whitespace. + Substituters are tried based on their Priority value, which each substituter can set independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. @@ -688,7 +705,8 @@ public: Setting<StringSet> trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of URLs of substituters, separated by whitespace. These are + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), + separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying `--option substituters urls` on the command line. Unprivileged users are only allowed to pass a subset of the @@ -817,8 +835,22 @@ public: > `.netrc`. )"}; - /* Path to the SSL CA file used */ - Path caFile; + Setting<Path> caFile{ + this, getDefaultSSLCertFile(), "ssl-cert-file", + R"( + The path of a file containing CA certificates used to + authenticate `https://` downloads. Nix by default will use + the first of the following files that exists: + + 1. `/etc/ssl/certs/ca-certificates.crt` + 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + + The path can be overridden by the following environment + variables, in order of precedence: + + 1. `NIX_SSL_CERT_FILE` + 2. `SSL_CERT_FILE` + )"}; #if __linux__ Setting<bool> filterSyscalls{ @@ -923,13 +955,6 @@ public: are loaded as plugins (non-recursively). )"}; - Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; - - bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - - void requireExperimentalFeature(const ExperimentalFeature &); - Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1479822a9..238fd1d98 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -12,7 +12,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const std::string name() override { return "Http Binary Cache Store"; } + const std::string name() override { return "HTTP Binary Cache Store"; } + + std::string doc() override + { + return + #include "http-binary-cache-store.md" + ; + } }; class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md new file mode 100644 index 000000000..20c26d0c2 --- /dev/null +++ b/src/libstore/http-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `http://...`, `https://...` + +This store allows a binary cache to be accessed via the HTTP +protocol. + +)" diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2c9dd2680..98322b045 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -12,17 +13,24 @@ namespace nix { -struct LegacySSHStoreConfig : virtual StoreConfig +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { - using StoreConfig::StoreConfig; - const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; - - const std::string name() override { return "Legacy SSH Store"; } + using CommonSSHStoreConfig::CommonSSHStoreConfig; + + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + std::string doc() override + { + return + #include "legacy-ssh-store.md" + ; + } }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store @@ -51,6 +59,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , CommonSSHStoreConfig(params) , LegacySSHStoreConfig(params) , Store(params) , host(host) diff --git a/src/libstore/legacy-ssh-store.md b/src/libstore/legacy-ssh-store.md new file mode 100644 index 000000000..043acebd6 --- /dev/null +++ b/src/libstore/legacy-ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh://[username@]hostname` + +This store type allows limited access to a remote store on another +machine via SSH. + +)" diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f20b1fa02..e5ee6fc15 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,6 +11,13 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig using BinaryCacheStoreConfig::BinaryCacheStoreConfig; const std::string name() override { return "Local Binary Cache Store"; } + + std::string doc() override + { + return + #include "local-binary-cache-store.md" + ; + } }; class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore diff --git a/src/libstore/local-binary-cache-store.md b/src/libstore/local-binary-cache-store.md new file mode 100644 index 000000000..93fddc840 --- /dev/null +++ b/src/libstore/local-binary-cache-store.md @@ -0,0 +1,16 @@ +R"( + +**Store URL format**: `file://`*path* + +This store allows reading and writing a binary cache stored in *path* +in the local filesystem. If *path* does not exist, it will be created. + +For example, the following builds or downloads `nixpkgs#hello` into +the local store and then copies it to the binary cache in +`/tmp/binary-cache`: + +``` +# nix copy --to file:///tmp/binary-cache nixpkgs#hello +``` + +)" diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 947707341..1e7f31a09 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "gc-store.hh" @@ -9,20 +10,28 @@ namespace nix { struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. + const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; + "root", + "Directory prefixed to all other paths."}; + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; + "state", + "Directory where Nix will store state."}; + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; + "log", + "directory where Nix will store log files."}; + const PathSetting realStoreDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"}; + "Physical path of the Nix store."}; }; class LocalFSStore : public virtual LocalFSStoreConfig, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 82edaa9bf..f58d90895 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -44,6 +44,13 @@ namespace nix { +std::string LocalStoreConfig::doc() +{ + return + #include "local-store.md" + ; +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -280,7 +287,7 @@ LocalStore::LocalStore(const Params & params) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); } else if (curSchema < nixSchemaVersion) { @@ -329,14 +336,14 @@ LocalStore::LocalStore(const Params & params) txn.commit(); } - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); lockFile(globalLock.get(), ltRead, true); } else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } @@ -366,7 +373,7 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( insert into Realisations (drvPath, outputName, outputPath, signatures) @@ -413,6 +420,13 @@ LocalStore::LocalStore(const Params & params) } +LocalStore::LocalStore(std::string scheme, std::string path, const Params & params) + : LocalStore(params) +{ + throw UnimplementedError("LocalStore"); +} + + AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; @@ -754,7 +768,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else @@ -763,7 +777,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); retrySQLite<void>([&]() { auto state(_state.lock()); if (auto oldR = queryRealisation_(*state, info.id)) { @@ -1052,7 +1066,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; auto drv = readInvalidDerivation(path); @@ -1120,54 +1134,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast<const NarInfo>( - std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -1560,7 +1526,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printInfo(format("reading the Nix store...")); + printInfo("reading the Nix store..."); bool errors = false; @@ -1950,5 +1916,6 @@ std::optional<std::string> LocalStore::getVersion() return nixVersion; } +static RegisterStoreImplementation<LocalStore, LocalStoreConfig> regLocalStore; } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a84eb7c26..1b668b6fd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sqlite.hh" @@ -38,11 +39,13 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Setting<bool> requireSigs{(StoreConfig*) this, settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; + "require-sigs", + "Whether store paths copied into this store should have a trusted signature."}; const std::string name() override { return "Local Store"; } -}; + std::string doc() override; +}; class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { @@ -100,9 +103,13 @@ public: /* Initialise the local store, upgrading the schema if necessary. */ LocalStore(const Params & params); + LocalStore(std::string scheme, std::string path, const Params & params); ~LocalStore(); + static std::set<std::string> uriSchemes() + { return {}; } + /* Implementations of abstract store API methods. */ std::string getUri() override; @@ -127,9 +134,6 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; diff --git a/src/libstore/local-store.md b/src/libstore/local-store.md new file mode 100644 index 000000000..8174df839 --- /dev/null +++ b/src/libstore/local-store.md @@ -0,0 +1,39 @@ +R"( + +**Store URL format**: `local`, *root* + +This store type accesses a Nix store in the local filesystem directly +(i.e. not via the Nix daemon). *root* is an absolute path that is +prefixed to other directories such as the Nix store directory. The +store pseudo-URL `local` denotes a store that uses `/` as its root +directory. + +A store that uses a *root* other than `/` is called a *chroot +store*. With such stores, the store directory is "logically" still +`/nix/store`, so programs stored in them can only be built and +executed by `chroot`-ing into *root*. Chroot stores only support +building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are +enabled. + +For example, the following uses `/tmp/root` as the chroot environment +to build or download `nixpkgs#hello` and then execute it: + +```console +# nix run --store /tmp/root nixpkgs#hello +Hello, world! +``` + +Here, the "physical" store location is `/tmp/root/nix/store`, and +Nix's store metadata is in `/tmp/root/nix/var/nix/db`. + +It is also possible, but not recommended, to change the "logical" +location of the Nix store from its default of `/nix/store`. This makes +it impossible to use default substituters such as +`https://cache.nixos.org/`, and thus you may have to build everything +locally. Here is an example: + +```console +# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello +``` + +)" diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 4fe1fcf56..7202a64b3 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -129,7 +129,7 @@ struct AutoUserLock : UserLock useUserNamespace = false; #endif - settings.requireExperimentalFeature(Xp::AutoAllocateUids); + experimentalFeatureSettings.require(Xp::AutoAllocateUids); assert(settings.startId > 0); assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max()); diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 7f1934510..7be3ba314 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index e4d95bab6..7aeec73b2 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 834626de9..1adeaf1f0 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index c4a66ed41..2ce6ec7bc 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index b28768459..89148d415 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -326,7 +326,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, throw Error( "the derivation '%s' doesn't have an output named '%s'", store.printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { DrvOutput outputId { *outputHash, output }; auto realisation = store.queryRealisation(outputId); if (!realisation) diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 3977fc6cc..d82b99bb4 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <memory> diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a0003588..f0dfcb19b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -275,6 +275,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case FSAccessor::Type::tMissing: default: throw Error("path '%s' does not exist in NAR", path); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 7d998ae0b..940e537b6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <functional> diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 4877f56d8..c596f2d71 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "nar-info.hh" diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 01683ec73..3cae8e659 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4d2781180..4a79cf4a1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() } if (errno) throw SysError("reading directory '%1%'", linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, "loaded %1% hash inodes", inodeHash.size()); return inodeHash; } @@ -73,7 +73,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { - debug(format("'%1%' is already linked") % dirent->d_name); + debug("'%1%' is already linked", dirent->d_name); continue; } @@ -102,7 +102,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { - debug(format("'%1%' is not allowed to be linked in macOS") % path); + debug("'%1%' is not allowed to be linked in macOS", path); return; } #endif @@ -146,7 +146,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); @@ -196,11 +196,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { - debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + debug("'%1%' is already linked to '%2%'", path, linkPath); return; } - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -213,8 +213,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % random()).str(); + Path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), random()); if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (errno == EMLINK) { @@ -222,7 +221,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo(format("'%1%' has maximum number of links") % linkPath); + printInfo("'%1%' has maximum number of links", linkPath); return; } throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 46bc35ebc..5a726fe90 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <cassert> #include <optional> @@ -9,6 +10,9 @@ namespace nix { +/** + * A non-empty set of outputs, specified by name + */ struct OutputNames : std::set<std::string> { using std::set<std::string>::set; @@ -18,6 +22,9 @@ struct OutputNames : std::set<std::string> { : std::set<std::string>(s) { assert(!empty()); } + /** + * Needs to be "inherited manually" + */ OutputNames(std::set<std::string> && s) : std::set<std::string>(s) { assert(!empty()); } @@ -28,6 +35,9 @@ struct OutputNames : std::set<std::string> { OutputNames() = delete; }; +/** + * The set of all outputs, without needing to name them explicitly + */ struct AllOutputs : std::monostate { }; typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw; @@ -36,7 +46,9 @@ struct OutputsSpec : _OutputsSpecRaw { using Raw = _OutputsSpecRaw; using Raw::Raw; - /* Force choosing a variant */ + /** + * Force choosing a variant + */ OutputsSpec() = delete; using Names = OutputNames; @@ -52,14 +64,20 @@ struct OutputsSpec : _OutputsSpecRaw { bool contains(const std::string & output) const; - /* Create a new OutputsSpec which is the union of this and that. */ + /** + * Create a new OutputsSpec which is the union of this and that. + */ OutputsSpec union_(const OutputsSpec & that) const; - /* Whether this OutputsSpec is a subset of that. */ + /** + * Whether this OutputsSpec is a subset of that. + */ bool isSubsetOf(const OutputsSpec & outputs) const; - /* Parse a string of the form 'output1,...outputN' or - '*', returning the outputs spec. */ + /** + * Parse a string of the form 'output1,...outputN' or '*', returning + * the outputs spec. + */ static OutputsSpec parse(std::string_view s); static std::optional<OutputsSpec> parseOpt(std::string_view s); @@ -81,8 +99,10 @@ struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { return static_cast<const Raw &>(*this); } - /* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the extended outputs spec. */ + /** + * Parse a string of the form 'prefix^output1,...outputN' or + * 'prefix^*', returning the prefix and the extended outputs spec. + */ static std::pair<std::string_view, ExtendedOutputsSpec> parse(std::string_view s); static std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> parseOpt(std::string_view s); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index bfb3857c0..71085a604 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a7fcbd232..b28bf751c 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "path.hh" diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 6893c3876..4f8dc4c1f 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 5d25656a5..a845b0e5f 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "path.hh" #include "derived-path.hh" diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 1e5579b90..4ca6747b3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string_view> @@ -8,13 +9,22 @@ namespace nix { struct Hash; +/** + * \ref StorePath "Store path" is the fundamental reference type of Nix. + * A store paths refers to a Store object. + * + * See glossary.html#gloss-store-path for more information on a + * conceptual level. + */ class StorePath { std::string baseName; public: - /* Size of the hash part of store paths, in base-32 characters. */ + /** + * Size of the hash part of store paths, in base-32 characters. + */ constexpr static size_t HashLen = 32; // i.e. 160 bits constexpr static size_t MaxPathLen = 211; @@ -45,8 +55,9 @@ public: return baseName != other.baseName; } - /* Check whether a file name ends with the extension for - derivations. */ + /** + * Check whether a file name ends with the extension for derivations. + */ bool isDerivation() const; std::string_view name() const @@ -67,7 +78,10 @@ public: typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -/* Extension of derivations in the Nix store. */ +/** + * The file extension of \ref Derivation derivations when serialized + * into store objects. + */ const std::string drvExtension = ".drv"; } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 42023cd0a..adc763e6a 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -96,7 +96,7 @@ bool PathLocks::lockPaths(const PathSet & paths, checkInterrupt(); Path lockPath = path + ".lock"; - debug(format("locking path '%1%'") % path); + debug("locking path '%1%'", path); AutoCloseFD fd; @@ -118,7 +118,7 @@ bool PathLocks::lockPaths(const PathSet & paths, } } - debug(format("lock acquired on '%1%'") % lockPath); + debug("lock acquired on '%1%'", lockPath); /* Check that the lock file hasn't become stale (i.e., hasn't been unlinked). */ @@ -130,7 +130,7 @@ bool PathLocks::lockPaths(const PathSet & paths, a lock on a deleted file. This means that other processes may create and acquire a lock on `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); + debug("open lock file '%1%' has become stale", lockPath); else break; } @@ -163,7 +163,7 @@ void PathLocks::unlock() "error (ignored): cannot close lock file on '%1%'", i.second); - debug(format("lock released on '%1%'") % i.second); + debug("lock released on '%1%'", i.second); } fds.clear(); diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 5e3a734b4..ba8f2749b 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index c551c5f3e..ba5c8583f 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -64,7 +64,7 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { - Path prefix = (format("%1%-%2%") % profile % num).str(); + Path prefix = fmt("%1%-%2%", profile, num); outLink = prefix + "-link"; } @@ -269,7 +269,7 @@ void switchGeneration( void lockProfile(PathLocks & lock, const Path & profile) { - lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); + lock.lockPaths({profile}, fmt("waiting for lock on profile '%1%'", profile)); lock.setDeletion(true); } @@ -282,28 +282,48 @@ std::string optimisticLockProfile(const Path & profile) Path profilesDir() { - auto profileRoot = createNixStateDir() + "/profiles"; + auto profileRoot = + (getuid() == 0) + ? rootProfilesDir() + : createNixStateDir() + "/profiles"; createDirs(profileRoot); return profileRoot; } +Path rootProfilesDir() +{ + return settings.nixStateDir + "/profiles/per-user/root"; +} + Path getDefaultProfile() { Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { - auto profile = - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : profilesDir() + "/profile"; + auto profile = profilesDir() + "/profile"; if (!pathExists(profileLink)) { replaceSymlink(profile, profileLink); } + // Backwards compatibiliy measure: Make root's profile available as + // `.../default` as it's what NixOS and most of the init scripts expect + Path globalProfileLink = settings.nixStateDir + "/profiles/default"; + if (getuid() == 0 && !pathExists(globalProfileLink)) { + replaceSymlink(profile, globalProfileLink); + } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { return profileLink; } } +Path defaultChannelsDir() +{ + return profilesDir() + "/channels"; +} + +Path rootChannelsDir() +{ + return rootProfilesDir() + "/channels"; +} } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index fbf95b850..6a5965390 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "pathlocks.hh" @@ -68,13 +69,32 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); -/* Creates and returns the path to a directory suitable for storing the user’s - profiles. */ +/** + * Create and return the path to a directory suitable for storing the user’s + * profiles. + */ Path profilesDir(); -/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ - nix/profile if XDG Base Directory Support is enabled), and create if doesn't - exist */ +/** + * Return the path to the profile directory for root (but don't try creating it) + */ +Path rootProfilesDir(); + +/** + * Create and return the path to the file used for storing the users's channels + */ +Path defaultChannelsDir(); + +/** + * Return the path to the channel directory for root (but don't try creating it) + */ +Path rootChannelsDir(); + +/** + * Resolve the default profile (~/.nix-profile by default, + * $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled), + * and create if doesn't exist + */ Path getDefaultProfile(); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 48d0283de..a18cf2aa8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <variant> diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 3bb297fc8..345f4528b 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -39,8 +39,7 @@ static void search( if (!match) continue; std::string ref(s.substr(i, refLength)); if (hashes.erase(ref)) { - debug(format("found reference to '%1%' at offset '%2%'") - % ref % i); + debug("found reference to '%1%' at offset '%2%'", ref, i); seen.insert(ref); } ++i; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 6f381f96c..52d71b333 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "hash.hh" #include "path.hh" diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 99f5544ef..e2673b6f6 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fs-accessor.hh" #include "ref.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d1296627a..d24d83117 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -265,7 +265,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); - overrides.erase(settings.experimentalFeatures.name); + overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) @@ -876,7 +876,7 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults( "the derivation '%s' doesn't have an output named '%s'", printStorePath(bfd.drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 11d089cd2..f5f45f853 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <limits> #include <string> @@ -22,11 +23,13 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting<int> maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; - const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(), - "max-connection-age", "number of seconds to reuse a connection"}; + const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, + std::numeric_limits<unsigned int>::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; }; /* FIXME: RemoteStore is a misnomer - should be something like @@ -38,8 +41,6 @@ class RemoteStore : public virtual RemoteStoreConfig, { public: - virtual bool sameMachine() = 0; - RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh index a13cda312..f412d6a20 100644 --- a/src/libstore/repair-flag.hh +++ b/src/libstore/repair-flag.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 8d76eee99..ac82147ee 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -40,12 +40,12 @@ struct S3Error : public Error /* Helper: given an Outcome<R, E>, return R in case of success, or throw an exception in case of an error. */ template<typename R, typename E> -R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) +R && checkAws(std::string_view s, Aws::Utils::Outcome<R, E> && outcome) { if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - fs.s + ": " + outcome.GetError().GetMessage()); + s + ": " + outcome.GetError().GetMessage()); return outcome.GetResultWithOwnership(); } @@ -192,19 +192,72 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; - const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; + + const Setting<std::string> profile{(StoreConfig*) this, "", "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + + const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + "Compression method for `.narinfo` files."}; + + const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", + "Compression method for `.ls` files."}; + + const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + const Setting<bool> multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", + "Whether to use multi-part uploads."}; + const Setting<uint64_t> bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } + + std::string doc() override + { + return + #include "s3-binary-cache-store.md" + ; + } }; struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore @@ -430,9 +483,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::string marker; do { - debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); + debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); - auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, + auto res = checkAws(fmt("AWS error listing bucket '%s'", bucketName), s3Helper.client->ListObjects( Aws::S3::Model::ListObjectsRequest() .WithBucket(bucketName) @@ -441,8 +494,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto & contents = res.GetContents(); - debug(format("got %d keys, next marker '%s'") - % contents.size() % res.GetNextMarker()); + debug("got %d keys, next marker '%s'", + contents.size(), res.GetNextMarker()); for (auto object : contents) { auto & key = object.GetKey(); diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bce828b11..c62ea5147 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "binary-cache-store.hh" diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md new file mode 100644 index 000000000..70fe0eb09 --- /dev/null +++ b/src/libstore/s3-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `s3://`*bucket-name* + +This store allows reading and writing a binary cache stored in an AWS +S3 bucket. + +)" diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index cdb3e5908..f0aeb3bed 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if ENABLE_S3 diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 3f76baa82..553fd3a09 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1853731a2..86410f998 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <functional> #include <string> diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh new file mode 100644 index 000000000..c27a5d00f --- /dev/null +++ b/src/libstore/ssh-store-config.hh @@ -0,0 +1,29 @@ +#pragma once +///@file + +#include "store-api.hh" + +namespace nix { + +struct CommonSSHStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", + "Path to the SSH private key used to authenticate to the remote machine."}; + + const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + "The public host key of the remote machine."}; + + const Setting<bool> compress{(StoreConfig*) this, false, "compress", + "Whether to enable SSH compression."}; + + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )"}; +}; + +} diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index a1d4daafd..962221ad2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "store-api.hh" #include "remote-store.hh" #include "remote-fs-accessor.hh" @@ -8,17 +9,22 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { using RemoteStoreConfig::RemoteStoreConfig; + using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + "Path to the `nix-daemon` executable on the remote machine."}; - const std::string name() override { return "SSH Store"; } + const std::string name() override { return "Experimental SSH Store"; } + + std::string doc() override + { + return + #include "ssh-store.md" + ; + } }; class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore @@ -28,6 +34,7 @@ public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) , SSHStoreConfig(params) , Store(params) , RemoteStore(params) @@ -49,9 +56,6 @@ public: return *uriSchemes().begin() + "://" + host; } - bool sameMachine() override - { return false; } - // FIXME extend daemon protocol, move implementation to RemoteStore std::optional<std::string> getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md new file mode 100644 index 000000000..881537e71 --- /dev/null +++ b/src/libstore/ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a +remote machine. + +)" diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 69bfe3418..6f6deda51 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "finally.hh" namespace nix { @@ -35,6 +36,9 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } if (compress) args.push_back("-C"); + + args.push_back("-oPermitLocalCommand=yes"); + args.push_back("-oLocalCommand=echo started"); } std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command) @@ -49,6 +53,11 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string ProcessOptions options; options.dieWithParent = false; + if (!fakeSSH && !useMaster) { + logger->pause(); + } + Finally cleanup = [&]() { logger->resume(); }; + conn->sshPid = startProcess([&]() { restoreProcessContext(); @@ -86,6 +95,18 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string in.readSide = -1; out.writeSide = -1; + // Wait for the SSH connection to be established, + // So that we don't overwrite the password prompt with our progress bar. + if (!fakeSSH && !useMaster) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH connection to '%s'", host); + } + conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); @@ -109,6 +130,9 @@ Path SSHMaster::startMaster() ProcessOptions options; options.dieWithParent = false; + logger->pause(); + Finally cleanup = [&]() { logger->resume(); }; + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -117,11 +141,7 @@ Path SSHMaster::startMaster() if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; + Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index dabbcedda..c86a8a986 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "sync.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 601efa1cc..8fc3ed46a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -445,10 +445,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); - if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) res.insert("recursive-nix"); return res; @@ -507,6 +507,54 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { @@ -790,13 +838,13 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, if (showHash) { s += info->narHash.to_string(Base16, false) + "\n"; - s += (format("%1%\n") % info->narSize).str(); + s += fmt("%1%\n", info->narSize); } auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; - s += (format("%1%\n") % info->references.size()).str(); + s += fmt("%1%\n", info->references.size()); for (auto & j : info->references) s += printStorePath(j) + "\n"; @@ -855,6 +903,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, auto info = queryPathInfo(storePath); jsonPath["path"] = printStorePath(info->path); + jsonPath["valid"] = true; jsonPath["narHash"] = info->narHash.to_string(hashBase, true); jsonPath["narSize"] = info->narSize; @@ -1016,7 +1065,7 @@ std::map<StorePath, StorePath> copyPaths( for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if<Realisation>(&path.raw)) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); toplevelRealisations.insert(*realisation); } } @@ -1100,6 +1149,9 @@ std::map<StorePath, StorePath> copyPaths( return storePathForDst; }; + // total is accessed by each copy, which are each handled in separate threads + std::atomic<uint64_t> total = 0; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); @@ -1120,7 +1172,13 @@ std::map<StorePath, StorePath> copyPaths( {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - srcStore.narFromPath(missingPath, sink); + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); + act.progress(total, info->narSize); + }); + TeeSink tee { sink, progressSink }; + + srcStore.narFromPath(missingPath, tee); }); pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } @@ -1241,7 +1299,7 @@ std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path) } } - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations) || !isValidPath(path)) return path; auto drv = readDerivation(path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d8db3596..4d1047380 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nar-info.hh" #include "realisation.hh" @@ -55,7 +56,10 @@ namespace nix { */ MakeError(SubstError, Error); -MakeError(BuildError, Error); // denotes a permanent build failure +/** + * denotes a permanent build failure + */ +MakeError(BuildError, Error); MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); @@ -78,7 +82,9 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Magic header of exportPath() output (obsolete). */ +/** + * Magic header of exportPath() output (obsolete). + */ const uint32_t exportMagic = 0x4558494e; @@ -101,17 +107,41 @@ struct StoreConfig : public Config virtual const std::string name() = 0; + virtual std::string doc() + { + return ""; + } + const PathSetting storeDir_{this, false, settings.nixStore, - "store", "path to the Nix store"}; + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; const Path storeDir = storeDir_; - const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size", + "Size of the in-memory store path metadata cache."}; - const Setting<bool> isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + const Setting<bool> isTrusted{this, false, "trusted", + R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )"}; - Setting<int> priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; + Setting<int> priority{this, 0, "priority", + R"( + Priority of this store when used as a substituter. A lower value means a higher priority. + )"}; - Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting<bool> wantMassQuery{this, false, "want-mass-query", + R"( + Whether this store (when used as a substituter) can be + queried efficiently for path validity. + )"}; Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), "system-features", @@ -125,23 +155,30 @@ public: typedef std::map<std::string, std::string> Params; - - protected: struct PathInfoCacheValue { - // Time of cache entry creation or update + /** + * Time of cache entry creation or update + */ std::chrono::time_point<std::chrono::steady_clock> time_point = std::chrono::steady_clock::now(); - // Null if missing + /** + * Null if missing + */ std::shared_ptr<const ValidPathInfo> value; - // Whether the value is valid as a cache entry. The path may not exist. + /** + * Whether the value is valid as a cache entry. The path may not + * exist. + */ bool isKnownNow(); - // Past tense, because a path can only be assumed to exists when - // isKnownNow() && didExist() + /** + * Past tense, because a path can only be assumed to exists when + * isKnownNow() && didExist() + */ inline bool didExist() { return value != nullptr; } @@ -175,35 +212,53 @@ public: std::string printStorePath(const StorePath & path) const; - // FIXME: remove + /** + * Deprecated + * + * \todo remove + */ StorePathSet parseStorePathSet(const PathSet & paths) const; PathSet printStorePathSet(const StorePathSet & path) const; - /* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const StorePathSet & paths); - /* Return true if ‘path’ is in the Nix store (but not the Nix - store itself). */ + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ bool isInStore(PathView path) const; - /* Return true if ‘path’ is a store path, i.e. a direct child of - the Nix store. */ + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ bool isStorePath(std::string_view path) const; - /* Split a path like /nix/store/<hash>-<name>/<bla> into - /nix/store/<hash>-<name> and /<bla>. */ + /** + * Split a path like /nix/store/<hash>-<name>/<bla> into + * /nix/store/<hash>-<name> and /<bla>. + */ std::pair<StorePath, Path> toStorePath(PathView path) const; - /* Follow symlinks until we end up with a path in the Nix store. */ + /** + * Follow symlinks until we end up with a path in the Nix store. + */ Path followLinksToStore(std::string_view path) const; - /* Same as followLinksToStore(), but apply toStorePath() to the - result. */ + /** + * Same as followLinksToStore(), but apply toStorePath() to the + * result. + */ StorePath followLinksToStorePath(std::string_view path) const; - /* Constructs a unique store path name. */ + /** + * Constructs a unique store path name. + */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; StorePath makeStorePath(std::string_view type, @@ -224,33 +279,40 @@ public: const StorePathSet & references = {}, bool hasSelfReference = false) const; - /* This is the preparatory part of addToStore(); it computes the - store path to which srcPath is to be copied. Returns the store - path and the cryptographic hash of the contents of srcPath. */ + /** + * Preparatory part of addToStore(). + * + * @return the store path to which srcPath is to be copied + * and the cryptographic hash of the contents of srcPath. + */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; - /* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ StorePath computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const; - /* Check whether a path is valid. */ + /** + * Check whether a path is valid. + */ bool isValidPath(const StorePath & path); protected: @@ -259,53 +321,68 @@ protected: public: - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ + /** + * If requested, substitute missing paths. This + * implements nix-copy-closure's --use-substitutes + * flag. + */ void substitutePaths(const StorePathSet & paths); - /* Query which of the given paths is valid. Optionally, try to - substitute missing paths. */ + /** + * Query which of the given paths is valid. Optionally, try to + * substitute missing paths. + */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); - /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be replaced by 'x' - (i.e. you'll get /nix/store/<hash>-x rather than - /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the - full store path. FIXME: should return a set of - std::variant<StorePath, HashPart> to get rid of this hack. */ + /** + * Query the set of all valid paths. Note that for some store + * backends, the name part of store paths may be replaced by 'x' + * (i.e. you'll get /nix/store/<hash>-x rather than + * /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the + * full store path. FIXME: should return a set of + * std::variant<StorePath, HashPart> to get rid of this hack. + */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } constexpr static const char * MissingName = "x"; - /* Query information about a valid path. It is permitted to omit - the name part of the store path. */ + /** + * Query information about a valid path. It is permitted to omit + * the name part of the store path. + */ ref<const ValidPathInfo> queryPathInfo(const StorePath & path); - /* Asynchronous version of queryPathInfo(). */ + /** + * Asynchronous version of queryPathInfo(). + */ void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; - /* Query the information about a realisation. */ + /** + * Query the information about a realisation. + */ std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &); - /* Asynchronous version of queryRealisation(). */ + /** + * Asynchronous version of queryRealisation(). + */ void queryRealisation(const DrvOutput &, Callback<std::shared_ptr<const Realisation>> callback) noexcept; - /* Check whether the given valid path info is sufficiently attested, by - either being signed by a trusted public key or content-addressed, in - order to be included in the given store. - - These same checks would be performed in addToStore, but this allows an - earlier failure in the case where dependencies need to be added too, but - the addToStore wouldn't fail until those dependencies are added. Also, - we don't really want to add the dependencies listed in a nar info we - don't trust anyyways. - */ + /** + * Check whether the given valid path info is sufficiently attested, by + * either being signed by a trusted public key or content-addressed, in + * order to be included in the given store. + * + * These same checks would be performed in addToStore, but this allows an + * earlier failure in the case where dependencies need to be added too, but + * the addToStore wouldn't fail until those dependencies are added. Also, + * we don't really want to add the dependencies listed in a nar info we + * don't trust anyyways. + */ virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; @@ -325,53 +402,77 @@ protected: public: - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ + /** + * Queries the set of incoming FS references for a store path. + * The result is not cleared. + */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ + /** + * @return all currently valid derivations that have `path' as an + * output. + * + * (Note that the result of `queryDeriver()' is the derivation that + * was actually used to produce `path', which may not exist + * anymore.) + */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; - /* Query the outputs of the derivation denoted by `path'. */ + /** + * Query the outputs of the derivation denoted by `path'. + */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName => outputPath for the given derivation. All - outputs are mentioned so ones mising the mapping are mapped to - `std::nullopt`. */ + /** + * Query the mapping outputName => outputPath for the given + * derivation. All outputs are mentioned so ones mising the mapping + * are mapped to `std::nullopt`. + */ virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. - Assume every output has a mapping and throw an exception otherwise. */ + /** + * Query the mapping outputName=>outputPath for the given derivation. + * Assume every output has a mapping and throw an exception otherwise. + */ OutputPathMap queryDerivationOutputMap(const StorePath & path); - /* Query the full store path given the hash part of a valid store - path, or empty if the path doesn't exist. */ + /** + * Query the full store path given the hash part of a valid store + * path, or empty if the path doesn't exist. + */ virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; - /* Query which of the given paths have substitutes. */ + /** + * Query which of the given paths have substitutes. + */ virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; - /* Query substitute info (i.e. references, derivers and download - sizes) of a map of paths to their optional ca values. The info - of the first succeeding substituter for each path will be - returned. If a path does not have substitute info, it's omitted - from the resulting ‘infos’ map. */ + /** + * Query substitute info (i.e. references, derivers and download + * sizes) of a map of paths to their optional ca values. The info of + * the first succeeding substituter for each path will be returned. + * If a path does not have substitute info, it's omitted from the + * resulting ‘infos’ map. + */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); - /* Import a path into the store. */ + /** + * Import a path into the store. + */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; - // A list of paths infos along with a source providing the content of the - // associated store path + /** + * A list of paths infos along with a source providing the content + * of the associated store path + */ using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>; - /* Import multiple paths into the store. */ + /** + * Import multiple paths into the store. + */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, @@ -383,10 +484,14 @@ public: RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path. + * + * @return The resulting path is returned. + * @param filter This function can be used to exclude files (see + * libutil/archive.hh). + */ virtual StorePath addToStore( std::string_view name, const Path & srcPath, @@ -396,26 +501,33 @@ public: RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); - /* Copy the contents of a path to the store and register the - validity the resulting path, using a constant amount of - memory. */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path, using a constant amount of + * memory. + */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, std::optional<Hash> expectedCAHash = {}); - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). - `dump` may be drained */ - // FIXME: remove? + /** + * Like addToStore(), but the contents of the path are contained + * in `dump', which is either a NAR serialisation (if recursive == + * true) or simply the contents of a regular file (if recursive == + * false). + * `dump` may be drained + * + * \todo remove? + */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ + /** + * Like addToStore, but the contents written to the output path is a + * regular file containing the given string. + */ virtual StorePath addTextToStore( std::string_view name, std::string_view s, @@ -436,140 +548,180 @@ public: virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) { return registerDrvOutput(output); } - /* Write a NAR dump of a store path. */ + /** + * Write a NAR dump of a store path. + */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ virtual void buildPaths( const std::vector<DerivedPath> & paths, BuildMode buildMode = bmNormal, std::shared_ptr<Store> evalStore = nullptr); - /* Like `buildPaths()`, but return a vector of `BuildResult`s - corresponding to each element in `paths`. Note that in case of - a build/substitution error, this function won't throw an - exception, but return a `BuildResult` containing an error - message. */ + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ virtual std::vector<BuildResult> buildPathsWithResults( const std::vector<DerivedPath> & paths, BuildMode buildMode = bmNormal, std::shared_ptr<Store> evalStore = nullptr); - /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). - - ‘drvPath’ is used to deduplicate worker goals so it is imperative that - is correct. That said, it doesn't literally need to be store path that - would be calculated from writing this derivation to the store: it is OK - if it instead is that of a Derivation which would resolve to this (by - taking the outputs of it's input derivations and adding them as input - sources) such that the build time referenceable-paths are the same. - - In the input-addressed case, we usually *do* use an "original" - unresolved derivations's path, as that is what will be used in the - `buildPaths` case. Also, the input-addressed output paths are verified - only by that contents of that specific unresolved derivation, so it is - nice to keep that information around so if the original derivation is - ever obtained later, it can be verified whether the trusted user in fact - used the proper output path. - - In the content-addressed case, we want to always use the - resolved drv path calculated from the provided derivation. This serves - two purposes: - - - It keeps the operation trustless, by ruling out a maliciously - invalid drv path corresponding to a non-resolution-equivalent - derivation. - - - For the floating case in particular, it ensures that the derivation - to output mapping respects the resolution equivalence relation, so - one cannot choose different resolution-equivalent derivations to - subvert dependency coherence (i.e. the property that one doesn't end - up with multiple different versions of dependencies without - explicitly choosing to allow it). - */ + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ virtual void ensurePath(const StorePath & path); - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ + /** + * Add a store path as a temporary root of the garbage collector. + * The root disappears as soon as we exit. + */ virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ + /** + * @return a string representing information about the path that + * can be loaded into the database using `nix-store --load-db' or + * `nix-store --register-validity'. + */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /* Write a JSON representation of store path metadata, such as the - hash and the references. If ‘includeImpureInfo’ is true, - variable elements such as the registration time are - included. If ‘showClosureSize’ is true, the closure size of - each path is included. */ + /** + * Write a JSON representation of store path metadata, such as the + * hash and the references. + * + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); - /* Return the size of the closure of the specified path, that is, - the sum of the size of the NAR serialisation of each path in - the closure. */ + /** + * @return the size of the closure of the specified path, that is, + * the sum of the size of the NAR serialisation of each path in the + * closure. + */ std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath); - /* Optimise the disk space usage of the Nix store by hard-linking files - with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking files + * with the same contents. + */ virtual void optimiseStore() { }; - /* Check the integrity of the Nix store. Returns true if errors - remain. */ + /** + * Check the integrity of the Nix store. + * + * @return true if errors remain. + */ virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; - /* Return an object to access files in the Nix store. */ + /** + * @return An object to access files in the Nix store. + */ virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); } - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ virtual void repairPath(const StorePath & path) { unsupported("repairPath"); } - /* Add signatures to the specified store path. The signatures are - not verified. */ + /** + * Add signatures to the specified store path. The signatures are + * not verified. + */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ - /* Read a derivation, after ensuring its existence through - ensurePath(). */ + /** + * Read a derivation, after ensuring its existence through + * ensurePath(). + */ Derivation derivationFromPath(const StorePath & drvPath); - /* Read a derivation (which must already be valid). */ + /** + * Read a derivation (which must already be valid). + */ Derivation readDerivation(const StorePath & drvPath); - /* Read a derivation from a potentially invalid path. */ + /** + * Read a derivation from a potentially invalid path. + */ Derivation readInvalidDerivation(const StorePath & drvPath); - /* Place in `out' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `out' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ + /** + * @param [out] out Place in here the set of all store paths in the + * file system closure of `storePath'; that is, all paths than can + * be directly or indirectly reached from it. `out' is not cleared. + * + * @param flipDirection If true, the set of paths that can reach + * `storePath' is returned; that is, the closures under the + * `referrers' relation instead of the `references' relation is + * returned. + */ virtual void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); @@ -578,27 +730,34 @@ public: StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - /* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths - that will be substituted. */ + /** + * Given a set of paths that are to be built, return the set of + * derivations that will be built, and the set of output paths that + * will be substituted. + */ virtual void queryMissing(const std::vector<DerivedPath> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); - /* Sort a set of paths topologically under the references - relation. If p refers to q, then p precedes q in this list. */ + /** + * Sort a set of paths topologically under the references + * relation. If p refers to q, then p precedes q in this list. + */ StorePaths topoSortPaths(const StorePathSet & paths); - /* Export multiple paths in the format expected by ‘nix-store - --import’. */ + /** + * Export multiple paths in the format expected by ‘nix-store + * --import’. + */ void exportPaths(const StorePathSet & paths, Sink & sink); void exportPath(const StorePath & path, Sink & sink); - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. Optionally, the contents of the NARs are - preloaded into the specified FS accessor to speed up subsequent - access. */ + /** + * Import a sequence of NAR dumps created by exportPaths() into the + * Nix store. Optionally, the contents of the NARs are preloaded + * into the specified FS accessor to speed up subsequent access. + */ StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -620,8 +779,9 @@ public: const Stats & getStats(); - /* Computes the full closure of of a set of store-paths for e.g. - derivations that need this information for `exportReferencesGraph`. + /** + * Computes the full closure of of a set of store-paths for e.g. + * derivations that need this information for `exportReferencesGraph`. */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); @@ -632,18 +792,24 @@ public: */ std::optional<StorePath> getBuildDerivationPath(const StorePath &); - /* Hack to allow long-running processes like hydra-queue-runner to - occasionally flush their path info cache. */ + /** + * Hack to allow long-running processes like hydra-queue-runner to + * occasionally flush their path info cache. + */ void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } - /* Establish a connection to the store, for store types that have - a notion of connection. Otherwise this is a no-op. */ + /** + * Establish a connection to the store, for store types that have + * a notion of connection. Otherwise this is a no-op. + */ virtual void connect() { }; - /* Get the protocol version of this store or it's connection. */ + /** + * Get the protocol version of this store or it's connection. + */ virtual unsigned int getProtocol() { return 0; @@ -659,7 +825,7 @@ public: return toRealPath(printStorePath(storePath)); } - /* + /** * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) */ @@ -671,7 +837,13 @@ protected: Stats stats; - /* Unsupported methods. */ + /** + * Helper for methods that are not unsupported: this is used for + * default definitions for virtual methods that are meant to be overriden. + * + * \todo Using this should be a last resort. It is better to make + * the method "virtual pure" and/or move it to a subclass. + */ [[noreturn]] void unsupported(const std::string & op) { throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); @@ -680,7 +852,9 @@ protected: }; -/* Copy a path from one store to another. */ +/** + * Copy a path from one store to another. + */ void copyStorePath( Store & srcStore, Store & dstStore, @@ -689,12 +863,14 @@ void copyStorePath( CheckSigsFlag checkSigs = CheckSigs); -/* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order (i.e. - if A is a reference of B, then A is copied before B), but the set - of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ +/** + * Copy store paths from one store to another. The paths may be copied + * in parallel. They are copied in a topologically sorted order (i.e. if + * A is a reference of B, then A is copied before B), but the set of + * store paths is not automatically closed; use copyClosure() for that. + * + * @return a map of what each path was copied to the dstStore as. + */ std::map<StorePath, StorePath> copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set &, @@ -709,7 +885,9 @@ std::map<StorePath, StorePath> copyPaths( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +/** + * Copy the closure of `paths` from `srcStore` to `dstStore`. + */ void copyClosure( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, @@ -724,52 +902,61 @@ void copyClosure( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Remove the temporary roots file for this process. Any temporary - root becomes garbage after this point unless it has been registered - as a (permanent) root. */ +/** + * Remove the temporary roots file for this process. Any temporary + * root becomes garbage after this point unless it has been registered + * as a (permanent) root. + */ void removeTempRoots(); -/* Resolve the derived path completely, failing if any derivation output - is unknown. */ +/** + * Resolve the derived path completely, failing if any derivation output + * is unknown. + */ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/* Return a Store object to access the Nix store denoted by - ‘uri’ (slight misnomer...). Supported values are: - - * ‘local’: The Nix store in /nix/store and database in - /nix/var/nix/db, accessed directly. - - * ‘daemon’: The Nix store accessed via a Unix domain socket - connection to nix-daemon. - - * ‘unix://<path>’: The Nix store accessed via a Unix domain socket - connection to nix-daemon, with the socket located at <path>. - - * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - whether the user has write access to the local Nix - store/database. - - * ‘file://<path>’: A binary cache stored in <path>. - - * ‘https://<path>’: A binary cache accessed via HTTP. - - * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple - Storage Service. - - * ‘ssh://[user@]<host>’: A remote Nix store accessed by running - ‘nix-store --serve’ via SSH. - - You can pass parameters to the store implementation by appending - ‘?key=value&key=value&...’ to the URI. -*/ +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + * + * @param uri Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://<path>’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at <path>. + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://<path>’: A binary cache stored in <path>. + * + * - ‘https://<path>’: A binary cache accessed via HTTP. + * + * - ‘s3://<path>’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]<host>’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store implementation by appending + * ‘?key=value&key=value&...’ to the URI. + */ ref<Store> openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -/* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options. */ +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ std::list<ref<Store>> getDefaultSubstituters(); struct StoreFactory @@ -812,8 +999,10 @@ struct RegisterStoreImplementation }; -/* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ +/** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const PathSet & paths); @@ -822,7 +1011,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo( std::istream & str, std::optional<HashResult> hashGiven = std::nullopt); -/* Split URI into protocol+hierarchy part and its parameter set. */ +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh index ff62fc359..2473e72c5 100644 --- a/src/libstore/store-cast.hh +++ b/src/libstore/store-cast.hh @@ -1,9 +1,17 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** + * Helper to try downcasting a Store with a nice method if it fails. + * + * This is basically an alternative to the user-facing part of + * Store::unsupported that allows us to still have a nice message but + * better interface design. + */ template<typename T> T & require(Store & store) { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc new file mode 100644 index 000000000..12be8504d --- /dev/null +++ b/src/libstore/tests/derivation.cc @@ -0,0 +1,124 @@ +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> + +#include "derivations.hh" + +#include "tests/libstore.hh" + +namespace nix { + +class DerivationTest : public LibStoreTest +{ +}; + +#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ + TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ + } + +TEST_JSON(DerivationOutput, inputAddressed, + R"({ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFixed, + R"({ + "hashAlgo": "r:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFloating, + R"({ + "hashAlgo": "r:sha256" + })", + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, deferred, + R"({ })", + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, impure, + R"({ + "hashAlgo": "r:sha256", + "impure": true + })", + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(Derivation, impure, + R"({ + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ + "cat", + "dog" + ] + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + ({ + Derivation drv; + drv.inputSrcs = { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + } + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + drv; + })) + +#undef TEST_JSON + +} diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 3bc812440..506f3ccb1 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index 05397659b..ef93457b5 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include <gtest/gtest.h> #include <gmock/gmock.h> diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh index 2d455c817..ded331b33 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/src/libstore/tests/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh index d7f1a8988..21cb62310 100644 --- a/src/libstore/tests/path.hh +++ b/src/libstore/tests/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 5c38323cd..0fb7c38e9 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -26,9 +26,9 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) + const std::string scheme, + std::string socket_path, + const Params & params) : UDSRemoteStore(params) { path.emplace(socket_path); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index f8dfcca70..bd1dcb67c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "remote-store.hh" #include "local-fs-store.hh" @@ -15,6 +16,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon } const std::string name() override { return "Local Daemon Store"; } + + std::string doc() override + { + return + #include "uds-remote-store.md" + ; + } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore @@ -29,9 +37,6 @@ public: static std::set<std::string> uriSchemes() { return {"unix"}; } - bool sameMachine() override - { return true; } - ref<FSAccessor> getFSAccessor() override { return LocalFSStore::getFSAccessor(); } diff --git a/src/libstore/uds-remote-store.md b/src/libstore/uds-remote-store.md new file mode 100644 index 000000000..8df0bd6ff --- /dev/null +++ b/src/libstore/uds-remote-store.md @@ -0,0 +1,9 @@ +R"( + +**Store URL format**: `daemon`, `unix://`*path* + +This store type accesses a Nix store by talking to a Nix daemon +listening on the Unix domain socket *path*. The store pseudo-URL +`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`. + +)" diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 87088a3ac..697dd2b8c 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "serialise.hh" @@ -14,6 +15,10 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) +/** + * Enumeration of all the request types for the "worker protocol", used + * by unix:// and ssh-ng:// stores. + */ typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, @@ -74,7 +79,12 @@ typedef enum { class Store; struct Source; -/* To guide overloading */ +/** + * Used to guide overloading + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ template<typename T> struct Phantom {}; @@ -103,18 +113,19 @@ MAKE_WORKER_PROTO(X_, Y_); #undef X_ #undef Y_ -/* These use the empty string for the null case, relying on the fact - that the underlying types never serialize to the empty string. - - We do this instead of a generic std::optional<T> instance because - ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - the same reason, we don't have a std::variant<T..> instances (ordinal - tags 0...n). - - We could the generic instances and then these as specializations for - compatability, but that's proven a bit finnicky, and also makes the - worker protocol harder to implement in other languages where such - specializations may not be allowed. +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional<T> instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant<T..> instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. */ MAKE_WORKER_PROTO(, std::optional<StorePath>); MAKE_WORKER_PROTO(, std::optional<ContentAddress>); diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 2d82b54e7..7b6c3fcb5 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <nlohmann/json.hpp> #include "config.hh" diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 38305e71c..54721649c 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 0e2b9d12c..268a798d9 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -87,7 +87,7 @@ static time_t dump(const Path & path, Sink & sink, PathFilter & filter) std::string name(i.name); size_t pos = i.name.find(caseHackSuffix); if (pos != std::string::npos) { - debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); + debug("removing case hack suffix from '%1%'", path + "/" + i.name); name.erase(pos); } if (!unhacked.emplace(name, i.name).second) @@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) if (archiveSettings.useCaseHack) { auto i = names.find(name); if (i != names.end()) { - debug(format("case collision between '%1%' and '%2%'") % i->first % name); + debug("case collision between '%1%' and '%2%'", i->first, name); name += caseHackSuffix; name += std::to_string(++i->second); } else diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index e42dea540..60e33dd40 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -7,54 +8,71 @@ namespace nix { -/* dumpPath creates a Nix archive of the specified path. The format - is as follows: - - IF path points to a REGULAR FILE: - dump(path) = attrs( - [ ("type", "regular") - , ("contents", contents(path)) - ]) - - IF path points to a DIRECTORY: - dump(path) = attrs( - [ ("type", "directory") - , ("entries", concat(map(f, sort(entries(path))))) - ]) - where f(fn) = attrs( - [ ("name", fn) - , ("file", dump(path + "/" + fn)) - ]) - - where: - - attrs(as) = concat(map(attr, as)) + encN(0) - attrs((a, b)) = encS(a) + encS(b) - - encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) - - encN(n) = 64-bit little-endian encoding of n. - - contents(path) = the contents of a regular file. - - sort(strings) = lexicographic sort by 8-bit value (strcmp). - - entries(path) = the entries of a directory, without `.' and - `..'. - - `+' denotes string concatenation. */ - - +/** + * dumpPath creates a Nix archive of the specified path. + * + * @param path the file system data to dump. Dumping is recursive so if + * this is a directory we dump it and all its children. + * + * @param [out] sink The serialised archive is fed into this sink. + * + * @param filter Can be used to skip certain files. + * + * The format is as follows: + * + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) + * + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) + * + * where: + * + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) + * + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * + * encN(n) = 64-bit little-endian encoding of n. + * + * contents(path) = the contents of a regular file. + * + * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * + * entries(path) = the entries of a directory, without `.' and + * `..'. + * + * `+' denotes string concatenation. + */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -/* Same as `void dumpPath()`, but returns the last modified date of the path */ +/** + * Same as dumpPath(), but returns the last modified date of the path. + */ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/** + * Dump an archive with a single file with these contents. + * + * @param s Contents of the file. + */ void dumpString(std::string_view s, Sink & sink); -/* FIXME: fix this API, it sucks. */ +/** + * \todo Fix this API, it sucks. + */ struct ParseSink { virtual void createDirectory(const Path & path) { }; @@ -68,8 +86,10 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; -/* If the NAR archive contains a single file at top-level, then save - the contents of the file to `s'. Otherwise barf. */ +/** + * If the NAR archive contains a single file at top-level, then save + * the contents of the file to `s'. Otherwise barf. + */ struct RetrieveRegularNARSink : ParseSink { bool regular = true; @@ -97,7 +117,9 @@ void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); -/* Read a NAR from 'source' and write it to 'sink'. */ +/** + * Read a NAR from 'source' and write it to 'sink'. + */ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 35686a8aa..081dbeb28 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -52,7 +52,7 @@ std::shared_ptr<Completions> completions; std::string completionMarker = "___COMPLETE___"; -std::optional<std::string> needsCompletion(std::string_view s) +static std::optional<std::string> needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -120,6 +120,12 @@ void Args::parseCmdline(const Strings & _cmdline) if (!argsSeen) initialFlagsProcessed(); + + /* Now that we are done parsing, make sure that any experimental + * feature required by the flags is enabled */ + for (auto & f : flagExperimentalFeatures) + experimentalFeatureSettings.require(f); + } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) @@ -128,12 +134,18 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; + + if (auto & f = flag.experimentalFeature) + flagExperimentalFeatures.insert(*f); + std::vector<std::string> args; bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { if (flag.handler.arity == ArityAny || anyCompleted) break; - throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); + throw UsageError( + "flag '%s' requires %d argument(s), but only %d were given", + name, flag.handler.arity, n); } if (auto prefix = needsCompletion(*pos)) { anyCompleted = true; @@ -152,7 +164,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) + { + if (auto & f = flag->experimentalFeature) + flagExperimentalFeatures.insert(*f); completions->add("--" + name, flag->description); + } } return false; } @@ -172,7 +188,8 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (prefix == "-") { completions->add("--"); for (auto & [flagName, flag] : shortFlags) - completions->add(std::string("-") + flagName, flag->description); + if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) + completions->add(std::string("-") + flagName, flag->description); } } @@ -230,6 +247,11 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; + // TODO With C++23 use `std::optional::tranform` + if (auto & xp = flag->experimentalFeature) + j["experimental-feature"] = showExperimentalFeature(*xp); + else + j["experimental-feature"] = nullptr; flags[name] = std::move(j); } @@ -326,6 +348,11 @@ Strings argvToStrings(int argc, char * * argv) return args; } +std::optional<ExperimentalFeature> Command::experimentalFeature () +{ + return { Xp::NixCommand }; +} + MultiCommand::MultiCommand(const Commands & commands_) : commands(commands_) { @@ -389,6 +416,11 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); + // TODO With C++23 use `std::optional::tranform` + if (auto xp = command->experimentalFeature()) + cat["experimental-feature"] = showExperimentalFeature(*xp); + else + cat["experimental-feature"] = nullptr; cmds[name] = std::move(j); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 84866f12b..d90129796 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <iostream> #include <map> @@ -18,16 +19,22 @@ class Args { public: - /* Parse the command line, throwing a UsageError if something goes - wrong. */ + /** + * Parse the command line, throwing a UsageError if something goes + * wrong. + */ void parseCmdline(const Strings & cmdline); - /* Return a short one-line description of the command. */ + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } - /* Return documentation about this command, in Markdown format. */ + /** + * Return documentation about this command, in Markdown format. + */ virtual std::string doc() { return ""; } protected: @@ -117,6 +124,8 @@ protected: Handler handler; std::function<void(size_t, std::string_view)> completer; + std::optional<ExperimentalFeature> experimentalFeature; + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht); }; @@ -144,13 +153,17 @@ protected: std::set<std::string> hiddenCategories; - /* Called after all command line flags before the first non-flag - argument (if any) have been processed. */ + /** + * Called after all command line flags before the first non-flag + * argument (if any) have been processed. + */ virtual void initialFlagsProcessed() {} - /* Called after the command line has been processed if we need to generate - completions. Useful for commands that need to know the whole command line - in order to know what completions to generate. */ + /** + * Called after the command line has been processed if we need to generate + * completions. Useful for commands that need to know the whole command line + * in order to know what completions to generate. + */ virtual void completionHook() { } public: @@ -164,7 +177,9 @@ public: expectedArgs.emplace_back(std::move(arg)); } - /* Expect a string argument. */ + /** + * Expect a string argument. + */ void expectArg(const std::string & label, std::string * dest, bool optional = false) { expectArgs({ @@ -174,7 +189,9 @@ public: }); } - /* Expect 0 or more arguments. */ + /** + * Expect 0 or more arguments. + */ void expectArgs(const std::string & label, std::vector<std::string> * dest) { expectArgs({ @@ -188,30 +205,48 @@ public: friend class MultiCommand; MultiCommand * parent = nullptr; + +private: + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set<ExperimentalFeature> flagExperimentalFeatures; }; -/* A command is an argument parser that can be executed by calling its - run() method. */ +/** + * A command is an argument parser that can be executed by calling its + * run() method. + */ struct Command : virtual public Args { friend class MultiCommand; virtual ~Command() { } - virtual void prepare() { }; + /** + * Entry point to the command + */ virtual void run() = 0; typedef int Category; static constexpr Category catDefault = 0; + virtual std::optional<ExperimentalFeature> experimentalFeature (); + virtual Category category() { return catDefault; } }; typedef std::map<std::string, std::function<ref<Command>()>> Commands; -/* An argument parser that supports multiple subcommands, - i.e. ‘<command> <subcommand>’. */ +/** + * An argument parser that supports multiple subcommands, + * i.e. ‘<command> <subcommand>’. + */ class MultiCommand : virtual public Args { public: @@ -219,7 +254,9 @@ public: std::map<Command::Category, std::string> categories; - // Selected command, if any. + /** + * Selected command, if any. + */ std::optional<std::pair<std::string, ref<Command>>> command; MultiCommand(const Commands & commands); @@ -254,8 +291,6 @@ enum CompletionType { }; extern CompletionType completionType; -std::optional<std::string> needsCompletion(std::string_view s); - void completePath(size_t, std::string_view prefix); void completeDir(size_t, std::string_view prefix); diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index ef31794be..3710d1239 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,13 +1,16 @@ #pragma once +///@file #include <future> #include <functional> namespace nix { -/* A callback is a wrapper around a lambda that accepts a valid of - type T or an exception. (We abuse std::future<T> to pass the value or - exception.) */ +/** + * A callback is a wrapper around a lambda that accepts a valid of + * type T or an exception. (We abuse std::future<T> to pass the value or + * exception.) + */ template<typename T> class Callback { diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index b132b4262..ddf6db6d1 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -100,4 +100,30 @@ std::ostream & operator << (std::ostream & stream, const CanonPath & path) return stream; } +std::string CanonPath::makeRelative(const CanonPath & path) const +{ + auto p1 = begin(); + auto p2 = path.begin(); + + for (; p1 != end() && p2 != path.end() && *p1 == *p2; ++p1, ++p2) ; + + if (p1 == end() && p2 == path.end()) + return "."; + else if (p1 == end()) + return std::string(p2.remaining); + else { + std::string res; + while (p1 != end()) { + ++p1; + if (!res.empty()) res += '/'; + res += ".."; + } + if (p2 != path.end()) { + if (!res.empty()) res += '/'; + res += p2.remaining; + } + return res; + } +} + } diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 9d5984584..76e48c4f2 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string> #include <optional> @@ -8,28 +9,31 @@ namespace nix { -/* A canonical representation of a path. It ensures the following: - - - It always starts with a slash. - - - It never ends with a slash, except if the path is "/". - - - A slash is never followed by a slash (i.e. no empty components). - - - There are no components equal to '.' or '..'. - - Note that the path does not need to correspond to an actually - existing path, and there is no guarantee that symlinks are - resolved. -*/ +/** + * A canonical representation of a path. It ensures the following: + * + * - It always starts with a slash. + * + * - It never ends with a slash, except if the path is "/". + * + * - A slash is never followed by a slash (i.e. no empty components). + * + * - There are no components equal to '.' or '..'. + * + * Note that the path does not need to correspond to an actually + * existing path, and there is no guarantee that symlinks are + * resolved. + */ class CanonPath { std::string path; public: - /* Construct a canon path from a non-canonical path. Any '.', '..' - or empty components are removed. */ + /** + * Construct a canon path from a non-canonical path. Any '.', '..' + * or empty components are removed. + */ CanonPath(std::string_view raw); explicit CanonPath(const char * raw) @@ -44,9 +48,11 @@ public: static CanonPath root; - /* If `raw` starts with a slash, return - `CanonPath(raw)`. Otherwise return a `CanonPath` representing - `root + "/" + raw`. */ + /** + * If `raw` starts with a slash, return + * `CanonPath(raw)`. Otherwise return a `CanonPath` representing + * `root + "/" + raw`. + */ CanonPath(std::string_view raw, const CanonPath & root); bool isRoot() const @@ -58,8 +64,10 @@ public: const std::string & abs() const { return path; } - /* Like abs(), but return an empty string if this path is - '/'. Thus the returned string never ends in a slash. */ + /** + * Like abs(), but return an empty string if this path is + * '/'. Thus the returned string never ends in a slash. + */ const std::string & absOrEmpty() const { const static std::string epsilon; @@ -85,6 +93,9 @@ public: bool operator != (const Iterator & x) const { return remaining.data() != x.remaining.data(); } + bool operator == (const Iterator & x) const + { return !(*this != x); } + const std::string_view operator * () const { return remaining.substr(0, slash); } @@ -104,7 +115,9 @@ public: std::optional<CanonPath> parent() const; - /* Remove the last component. Panics if this path is the root. */ + /** + * Remove the last component. Panics if this path is the root. + */ void pop(); std::optional<std::string_view> dirOf() const @@ -125,10 +138,12 @@ public: bool operator != (const CanonPath & x) const { return path != x.path; } - /* Compare paths lexicographically except that path separators - are sorted before any other character. That is, in the sorted order - a directory is always followed directly by its children. For - instance, 'foo' < 'foo/bar' < 'foo!'. */ + /** + * Compare paths lexicographically except that path separators + * are sorted before any other character. That is, in the sorted order + * a directory is always followed directly by its children. For + * instance, 'foo' < 'foo/bar' < 'foo!'. + */ bool operator < (const CanonPath & x) const { auto i = path.begin(); @@ -144,28 +159,42 @@ public: return i == path.end() && j != x.path.end(); } - /* Return true if `this` is equal to `parent` or a child of - `parent`. */ + /** + * Return true if `this` is equal to `parent` or a child of + * `parent`. + */ bool isWithin(const CanonPath & parent) const; CanonPath removePrefix(const CanonPath & prefix) const; - /* Append another path to this one. */ + /** + * Append another path to this one. + */ void extend(const CanonPath & x); - /* Concatenate two paths. */ + /** + * Concatenate two paths. + */ CanonPath operator + (const CanonPath & x) const; - /* Add a path component to this one. It must not contain any slashes. */ + /** + * Add a path component to this one. It must not contain any slashes. + */ void push(std::string_view c); CanonPath operator + (std::string_view c) const; - /* Check whether access to this path is allowed, which is the case - if 1) `this` is within any of the `allowed` paths; or 2) any of - the `allowed` paths are within `this`. (The latter condition - ensures access to the parents of allowed paths.) */ + /** + * Check whether access to this path is allowed, which is the case + * if 1) `this` is within any of the `allowed` paths; or 2) any of + * the `allowed` paths are within `this`. (The latter condition + * ensures access to the parents of allowed paths.) + */ bool isAllowed(const std::set<CanonPath> & allowed) const; + + /* Return a representation `x` of `path` relative to `this`, i.e. + `CanonPath(this.makeRelative(x), this) == path`. */ + std::string makeRelative(const CanonPath & path) const; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh index d08c8ad29..574ae8e5b 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/cgroup.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if __linux__ @@ -18,10 +19,12 @@ struct CgroupStats std::optional<std::chrono::microseconds> cpuUser, cpuSystem; }; -/* Destroy the cgroup denoted by 'path'. The postcondition is that - 'path' does not exist, and thus any processes in the cgroup have - been killed. Also return statistics from the cgroup just before - destruction. */ +/** + * Destroy the cgroup denoted by 'path'. The postcondition is that + * 'path' does not exist, and thus any processes in the cgroup have + * been killed. Also return statistics from the cgroup just before + * destruction. + */ CgroupStats destroyCgroup(const Path & cgroup); } diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index 0a4f0b400..d914e2542 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <cstdint> #include <cstdlib> @@ -7,20 +8,24 @@ namespace nix { -/* Provides an indexable container like vector<> with memory overhead - guarantees like list<> by allocating storage in chunks of ChunkSize - elements instead of using a contiguous memory allocation like vector<> - does. Not using a single vector that is resized reduces memory overhead - on large data sets by on average (growth factor)/2, mostly - eliminates copies within the vector during resizing, and provides stable - references to its elements. */ +/** + * Provides an indexable container like vector<> with memory overhead + * guarantees like list<> by allocating storage in chunks of ChunkSize + * elements instead of using a contiguous memory allocation like vector<> + * does. Not using a single vector that is resized reduces memory overhead + * on large data sets by on average (growth factor)/2, mostly + * eliminates copies within the vector during resizing, and provides stable + * references to its elements. + */ template<typename T, size_t ChunkSize> class ChunkedVector { private: uint32_t size_ = 0; std::vector<std::vector<T>> chunks; - /* keep this out of the ::add hot path */ + /** + * Keep this out of the ::add hot path + */ [[gnu::noinline]] auto & addChunk() { diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh index 779b9b2d5..16e3b93e4 100644 --- a/src/libutil/closure.hh +++ b/src/libutil/closure.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include <set> #include <future> #include "sync.hh" diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index eecd5b819..2b5424b3d 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,8 @@ #pragma once +///@file -/* Awfull hacky generation of the comparison operators by doing a lexicographic +/** + * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * * ``` diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index c470b82a5..3892831c2 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "types.hh" diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh index 8ded295f9..093e7a915 100644 --- a/src/libutil/compute-levels.hh +++ b/src/libutil/compute-levels.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "types.hh" namespace nix { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index b349f2d80..8d63536d6 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -70,10 +70,17 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } +// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features. +static bool applicable(const Config::SettingData & sd) +{ + return !sd.isAlias + && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature); +} + void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly) { for (auto & opt : _settings) - if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) + if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -147,9 +154,8 @@ nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (!s.second.isAlias) { + if (applicable(s.second)) res.emplace(s.first, s.second.setting->toJSON()); - } return res; } @@ -157,24 +163,31 @@ std::string Config::toKeyValue() { auto res = std::string(); for (auto & s : _settings) - if (!s.second.isAlias) { + if (applicable(s.second)) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); - } return res; } void Config::convertToArgs(Args & args, const std::string & category) { - for (auto & s : _settings) + for (auto & s : _settings) { + /* We do include args for settings gated on disabled + experimental-features. The args themselves however will also be + gated on any experimental feature the underlying setting is. */ if (!s.second.isAlias) s.second.setting->convertToArg(args, category); + } } AbstractSetting::AbstractSetting( const std::string & name, const std::string & description, - const std::set<std::string> & aliases) - : name(name), description(stripIndentation(description)), aliases(aliases) + const std::set<std::string> & aliases, + std::optional<ExperimentalFeature> experimentalFeature) + : name(name) + , description(stripIndentation(description)) + , aliases(aliases) + , experimentalFeature(experimentalFeature) { } @@ -210,6 +223,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .category = category, .labels = {"value"}, .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, }); if (isAppendable()) @@ -219,6 +233,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .category = category, .labels = {"value"}, .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, }); } @@ -270,13 +285,15 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }} + .handler = {[this]() { override(true); }}, + .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }} + .handler = {[this]() { override(false); }}, + .experimentalFeature = experimentalFeature, }); } @@ -444,4 +461,30 @@ GlobalConfig::Register::Register(Config * config) configRegistrations->emplace_back(config); } +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const +{ + auto & f = experimentalFeatures.get(); + return std::find(f.begin(), f.end(), feature) != f.end(); +} + +void ExperimentalFeatureSettings::require(const ExperimentalFeature & feature) const +{ + if (!isEnabled(feature)) + throw MissingExperimentalFeature(feature); +} + +bool ExperimentalFeatureSettings::isEnabled(const std::optional<ExperimentalFeature> & feature) const +{ + return !feature || isEnabled(*feature); +} + +void ExperimentalFeatureSettings::require(const std::optional<ExperimentalFeature> & feature) const +{ + if (feature) require(*feature); +} + } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 7ac43c854..3c1d70294 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -1,12 +1,14 @@ +#pragma once +///@file + #include <cassert> #include <map> #include <set> -#include "types.hh" - #include <nlohmann/json_fwd.hpp> -#pragma once +#include "types.hh" +#include "experimental-features.hh" namespace nix { @@ -123,21 +125,21 @@ public: void reapplyUnknownSettings(); }; -/* A class to simplify providing configuration settings. The typical - use is to inherit Config and add Setting<T> members: - - class MyClass : private Config - { - Setting<int> foo{this, 123, "foo", "the number of foos to use"}; - Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"}; - - MyClass() : Config(readConfigFile("/etc/my-app.conf")) - { - std::cout << foo << "\n"; // will print 123 unless overridden - } - }; -*/ - +/** + * A class to simplify providing configuration settings. The typical + * use is to inherit Config and add Setting<T> members: + * + * class MyClass : private Config + * { + * Setting<int> foo{this, 123, "foo", "the number of foos to use"}; + * Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"}; + * + * MyClass() : Config(readConfigFile("/etc/my-app.conf")) + * { + * std::cout << foo << "\n"; // will print 123 unless overridden + * } + * }; + */ class Config : public AbstractConfig { friend class AbstractSetting; @@ -194,12 +196,15 @@ public: bool overridden = false; + std::optional<ExperimentalFeature> experimentalFeature; + protected: AbstractSetting( const std::string & name, const std::string & description, - const std::set<std::string> & aliases); + const std::set<std::string> & aliases, + std::optional<ExperimentalFeature> experimentalFeature = std::nullopt); virtual ~AbstractSetting() { @@ -224,7 +229,9 @@ protected: bool isOverridden() const { return overridden; } }; -/* A setting of type T. */ +/** + * A setting of type T. + */ template<typename T> class BaseSetting : public AbstractSetting { @@ -240,8 +247,9 @@ public: const bool documentDefault, const std::string & name, const std::string & description, - const std::set<std::string> & aliases = {}) - : AbstractSetting(name, description, aliases) + const std::set<std::string> & aliases = {}, + std::optional<ExperimentalFeature> experimentalFeature = std::nullopt) + : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) , defaultValue(def) , documentDefault(documentDefault) @@ -296,8 +304,9 @@ public: const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}, - const bool documentDefault = true) - : BaseSetting<T>(def, documentDefault, name, description, aliases) + const bool documentDefault = true, + std::optional<ExperimentalFeature> experimentalFeature = std::nullopt) + : BaseSetting<T>(def, documentDefault, name, description, aliases, experimentalFeature) { options->addSetting(this); } @@ -305,8 +314,10 @@ public: void operator =(const T & v) { this->assign(v); } }; -/* A special setting for Paths. These are automatically canonicalised - (e.g. "/foo//bar/" becomes "/foo/bar"). */ +/** + * A special setting for Paths. These are automatically canonicalised + * (e.g. "/foo//bar/" becomes "/foo/bar"). + */ class PathSetting : public BaseSetting<Path> { bool allowEmpty; @@ -357,4 +368,37 @@ struct GlobalConfig : public AbstractConfig extern GlobalConfig globalConfig; + +struct ExperimentalFeatureSettings : Config { + + Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + /** + * Check whether the given experimental feature is enabled. + */ + bool isEnabled(const ExperimentalFeature &) const; + + /** + * Require an experimental feature be enabled, throwing an error if it is + * not. + */ + void require(const ExperimentalFeature &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function returns true in that case. + */ + bool isEnabled(const std::optional<ExperimentalFeature> &) const; + + /** + * `std::nullopt` pointer means no feature, which means there is nothing that could be + * disabled, and so the function does nothing in that case. + */ + void require(const std::optional<ExperimentalFeature> &) const; +}; + +// FIXME: don't use a global variable. +extern ExperimentalFeatureSettings experimentalFeatureSettings; + } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index e4f0d4677..c9d61942a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -302,14 +302,14 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (!einfo.traces.empty()) { size_t count = 0; for (const auto & trace : einfo.traces) { + if (trace.hint.str().empty()) continue; + if (frameOnly && !trace.frame) continue; + if (!showTrace && count > 3) { oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; break; } - if (trace.hint.str().empty()) continue; - if (frameOnly && !trace.frame) continue; - count++; frameOnly = trace.frame; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0ebeaba61..eafc6a540 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "suggestions.hh" #include "ref.hh" @@ -54,20 +55,26 @@ typedef enum { lvlVomit } Verbosity; -// the lines of code surrounding an error. +/** + * The lines of code surrounding an error. + */ struct LinesOfCode { std::optional<std::string> prevLineOfCode; std::optional<std::string> errLineOfCode; std::optional<std::string> nextLineOfCode; }; -/* An abstract type that represents a location in a source file. */ +/** + * An abstract type that represents a location in a source file. + */ struct AbstractPos { uint32_t line = 0; uint32_t column = 0; - /* Return the contents of the source file. */ + /** + * Return the contents of the source file. + */ virtual std::optional<std::string> getSource() const { return std::nullopt; }; @@ -104,8 +111,10 @@ struct ErrorInfo { std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace); -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ +/** + * BaseError should generally not be caught, as it has Interrupted as + * a subclass. Catch Error instead. + */ class BaseError : public std::exception { protected: diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index ac372e03e..5948ad7ad 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "error.hh" @@ -12,7 +13,7 @@ namespace nix { * * If you update this, don’t forget to also change the map defining their * string representation in the corresponding `.cc` file. - **/ + */ enum struct ExperimentalFeature { CaDerivations, diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 3a732cff8..56be76ecc 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -15,9 +15,9 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, { tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); } Path createTempDir(const Path & tmpRoot, const Path & prefix, diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index dee2e8d2f..db654301f 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -1,6 +1,9 @@ #pragma once +///@file -/* A trivial class to run a function at the end of a scope. */ +/** + * A trivial class to run a function at the end of a scope. + */ template<typename Fn> class Finally { diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index e879fd3b8..727255b45 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <boost/format.hpp> #include <string> @@ -8,30 +9,25 @@ namespace nix { -/* Inherit some names from other namespaces for convenience. */ +/** + * Inherit some names from other namespaces for convenience. + */ using boost::format; -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ +/** + * A variadic template that does nothing. Useful to call a function + * for all variadic arguments but ignoring the result. + */ struct nop { template<typename... T> nop(T...) {} }; -struct FormatOrString -{ - std::string s; - FormatOrString(std::string s) : s(std::move(s)) { }; - template<class F> - FormatOrString(const F & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; -}; - - -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - +/** + * A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + * equivalent to ‘boost::format(format) % a_0 % ... % + * ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + * takes place). + */ template<class F> inline void formatHelper(F & f) { @@ -53,11 +49,6 @@ inline std::string fmt(const char * s) return s; } -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} - template<typename... Args> inline std::string fmt(const std::string & fs, const Args & ... args) { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index cb13ef0e5..bf2b9a286 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string> #include <string_view> @@ -8,21 +9,23 @@ namespace nix { namespace git { -// A line from the output of `git ls-remote --symref`. -// -// These can be of two kinds: -// -// - Symbolic references of the form -// -// ref: {target} {reference} -// -// where {target} is itself a reference and {reference} is optional -// -// - Object references of the form -// -// {target} {reference} -// -// where {target} is a commit id and {reference} is mandatory +/** + * A line from the output of `git ls-remote --symref`. + * + * These can be of two kinds: + * + * - Symbolic references of the form + * + * ref: {target} {reference} + * + * where {target} is itself a reference and {reference} is optional + * + * - Object references of the form + * + * {target} {reference} + * + * where {target} is a commit id and {reference} is mandatory + */ struct LsRemoteRefLine { enum struct Kind { Symbolic, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index d2fd0c15a..5735e4715 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -71,12 +71,13 @@ const std::string base16Chars = "0123456789abcdef"; static std::string printHash16(const Hash & hash) { - char buf[hash.hashSize * 2]; + std::string buf; + buf.reserve(hash.hashSize * 2); for (unsigned int i = 0; i < hash.hashSize; i++) { - buf[i * 2] = base16Chars[hash.hash[i] >> 4]; - buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + buf.push_back(base16Chars[hash.hash[i] >> 4]); + buf.push_back(base16Chars[hash.hash[i] & 0x0f]); } - return std::string(buf, hash.hashSize * 2); + return buf; } @@ -130,7 +131,7 @@ std::string Hash::to_string(Base base, bool includeType) const break; case Base64: case SRI: - s += base64Encode(std::string((const char *) hash, hashSize)); + s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } return s; @@ -403,7 +404,7 @@ HashType parseHashType(std::string_view s) throw UsageError("unknown hash algorithm '%1%'", s); } -std::string printHashType(HashType ht) +std::string_view printHashType(HashType ht) { switch (ht) { case htMD5: return "md5"; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 00f70a572..be1fdba2a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "serialise.hh" @@ -33,62 +34,86 @@ struct Hash HashType type; - /* Create a zero-filled hash object. */ + /** + * Create a zero-filled hash object. + */ Hash(HashType type); - /* Parse the hash from a string representation in the format - "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a - Subresource Integrity hash expression). If the 'type' argument - is not present, then the hash type must be specified in the - string. */ + /** + * Parse the hash from a string representation in the format + * "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a + * Subresource Integrity hash expression). If the 'type' argument + * is not present, then the hash type must be specified in the + * string. + */ static Hash parseAny(std::string_view s, std::optional<HashType> type); - /* Parse a hash from a string representation like the above, except the - type prefix is mandatory is there is no separate arguement. */ + /** + * Parse a hash from a string representation like the above, except the + * type prefix is mandatory is there is no separate arguement. + */ static Hash parseAnyPrefixed(std::string_view s); - /* Parse a plain hash that musst not have any prefix indicating the type. - The type is passed in to disambiguate. */ + /** + * Parse a plain hash that musst not have any prefix indicating the type. + * The type is passed in to disambiguate. + */ static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); static Hash parseSRI(std::string_view original); private: - /* The type must be provided, the string view must not include <type> - prefix. `isSRI` helps disambigate the various base-* encodings. */ + /** + * The type must be provided, the string view must not include <type> + * prefix. `isSRI` helps disambigate the various base-* encodings. + */ Hash(std::string_view s, HashType type, bool isSRI); public: - /* Check whether two hash are equal. */ + /** + * Check whether two hash are equal. + */ bool operator == (const Hash & h2) const; - /* Check whether two hash are not equal. */ + /** + * Check whether two hash are not equal. + */ bool operator != (const Hash & h2) const; - /* For sorting. */ + /** + * For sorting. + */ bool operator < (const Hash & h) const; - /* Returns the length of a base-16 representation of this hash. */ + /** + * Returns the length of a base-16 representation of this hash. + */ size_t base16Len() const { return hashSize * 2; } - /* Returns the length of a base-32 representation of this hash. */ + /** + * Returns the length of a base-32 representation of this hash. + */ size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } - /* Returns the length of a base-64 representation of this hash. */ + /** + * Returns the length of a base-64 representation of this hash. + */ size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } - /* Return a string representation of the hash, in base-16, base-32 - or base-64. By default, this is prefixed by the hash type - (e.g. "sha256:"). */ + /** + * Return a string representation of the hash, in base-16, base-32 + * or base-64. By default, this is prefixed by the hash type + * (e.g. "sha256:"). + */ std::string to_string(Base base, bool includeType) const; std::string gitRev() const @@ -104,36 +129,54 @@ public: static Hash dummy; }; -/* Helper that defaults empty hashes to the 0 hash. */ +/** + * Helper that defaults empty hashes to the 0 hash. + */ Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht); -/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +/** + * Print a hash in base-16 if it's MD5, or base-32 otherwise. + */ std::string printHash16or32(const Hash & hash); -/* Compute the hash of the given string. */ +/** + * Compute the hash of the given string. + */ Hash hashString(HashType ht, std::string_view s); -/* Compute the hash of the given file. */ +/** + * Compute the hash of the given file. + */ Hash hashFile(HashType ht, const Path & path); -/* Compute the hash of the given path. The hash is defined as - (essentially) hashString(ht, dumpPath(path)). */ +/** + * Compute the hash of the given path. The hash is defined as + * (essentially) hashString(ht, dumpPath(path)). + */ typedef std::pair<Hash, uint64_t> HashResult; HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); -/* Compress a hash to the specified number of bytes by cyclically - XORing bytes together. */ +/** + * Compress a hash to the specified number of bytes by cyclically + * XORing bytes together. + */ Hash compressHash(const Hash & hash, unsigned int newSize); -/* Parse a string representing a hash type. */ +/** + * Parse a string representing a hash type. + */ HashType parseHashType(std::string_view s); -/* Will return nothing on parse error */ +/** + * Will return nothing on parse error + */ std::optional<HashType> parseHashTypeOpt(std::string_view s); -/* And the reverse. */ -std::string printHashType(HashType ht); +/** + * And the reverse. + */ +std::string_view printHashType(HashType ht); union Ctx; diff --git a/src/libutil/hilite.hh b/src/libutil/hilite.hh index f8bdbfc55..2d5cf7c6f 100644 --- a/src/libutil/hilite.hh +++ b/src/libutil/hilite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <regex> #include <vector> @@ -6,11 +7,13 @@ namespace nix { -/* Highlight all the given matches in the given string `s` by wrapping - them between `prefix` and `postfix`. - - If some matches overlap, then their union will be wrapped rather - than the individual matches. */ +/** + * Highlight all the given matches in the given string `s` by wrapping + * them between `prefix` and `postfix`. + * + * If some matches overlap, then their union will be wrapped rather + * than the individual matches. + */ std::string hiliteMatches( std::string_view s, std::vector<std::smatch> matches, diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh index bd75748ad..b26163a04 100644 --- a/src/libutil/json-impls.hh +++ b/src/libutil/json-impls.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nlohmann/json_fwd.hpp" diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index b8a031227..eb00e954f 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <nlohmann/json.hpp> diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 904ba6ebe..5a2dd99af 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -32,7 +32,8 @@ void Logger::warn(const std::string & msg) void Logger::writeToStdout(std::string_view s) { - std::cout << s << "\n"; + writeFull(STDOUT_FILENO, s); + writeFull(STDOUT_FILENO, "\n"); } class SimpleLogger : public Logger @@ -53,7 +54,7 @@ public: return printBuildLogs; } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; @@ -64,14 +65,15 @@ public: switch (lvl) { case lvlError: c = '3'; break; case lvlWarn: c = '4'; break; - case lvlInfo: c = '5'; break; + case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; + default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; } - writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + writeToStderr(prefix + filterANSIEscapes(s, !tty) + "\n"); } void logEI(const ErrorInfo & ei) override @@ -84,7 +86,7 @@ public: void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) - override + override { if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); @@ -173,12 +175,12 @@ struct JSONLogger : Logger { prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace)); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { nlohmann::json json; json["action"] = "msg"; json["level"] = lvl; - json["msg"] = fs.s; + json["msg"] = s; write(json); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 4642c49f7..576068c22 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -72,14 +73,17 @@ public: virtual void stop() { }; + virtual void pause() { }; + virtual void resume() { }; + // Whether the logger prints the whole build log virtual bool isVerbose() { return false; } - virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + virtual void log(Verbosity lvl, std::string_view s) = 0; - void log(const FormatOrString & fs) + void log(std::string_view s) { - log(lvlInfo, fs); + log(lvlInfo, s); } virtual void logEI(const ErrorInfo & ei) = 0; @@ -102,11 +106,9 @@ public: virtual void writeToStdout(std::string_view s); template<typename... Args> - inline void cout(const std::string & fs, const Args & ... args) + inline void cout(const Args & ... args) { - boost::format f(fs); - formatHelper(f, args...); - writeToStdout(f.str()); + writeToStdout(fmt(args...)); } virtual std::optional<char> ask(std::string_view s) @@ -216,7 +218,9 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) -/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ +/** + * if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. + */ template<typename... Args> inline void warn(const std::string & fs, const Args & ... args) { diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 6ef4a3e06..0e19517ed 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <cassert> #include <map> @@ -7,7 +8,9 @@ namespace nix { -/* A simple least-recently used cache. Not thread-safe. */ +/** + * A simple least-recently used cache. Not thread-safe. + */ template<typename Key, typename Value> class LRUCache { @@ -31,7 +34,9 @@ public: LRUCache(size_t capacity) : capacity(capacity) { } - /* Insert or upsert an item in the cache. */ + /** + * Insert or upsert an item in the cache. + */ void upsert(const Key & key, const Value & value) { if (capacity == 0) return; @@ -39,7 +44,9 @@ public: erase(key); if (data.size() >= capacity) { - /* Retire the oldest item. */ + /** + * Retire the oldest item. + */ auto oldest = lru.begin(); data.erase(*oldest); lru.erase(oldest); @@ -63,14 +70,18 @@ public: return true; } - /* Look up an item in the cache. If it exists, it becomes the most - recently used item. */ + /** + * Look up an item in the cache. If it exists, it becomes the most + * recently used item. + * */ std::optional<Value> get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; - /* Move this item to the back of the LRU list. */ + /** + * Move this item to the back of the LRU list. + */ lru.erase(i->second.first.it); auto j = lru.insert(lru.end(), i); i->second.first.it = j; diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 9518cf8aa..86d0115fc 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <thread> #include <atomic> diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index e82379b9c..0b7eeb66c 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index d49067bb9..6247b6125 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <functional> #include <limits> @@ -11,33 +12,37 @@ namespace nix { -/* This template class implements a simple pool manager of resources - of some type R, such as database connections. It is used as - follows: - - class Connection { ... }; - - Pool<Connection> pool; - - { - auto conn(pool.get()); - conn->exec("select ..."); - } - - Here, the Connection object referenced by ‘conn’ is automatically - returned to the pool when ‘conn’ goes out of scope. -*/ - +/** + * This template class implements a simple pool manager of resources + * of some type R, such as database connections. It is used as + * follows: + * + * class Connection { ... }; + * + * Pool<Connection> pool; + * + * { + * auto conn(pool.get()); + * conn->exec("select ..."); + * } + * + * Here, the Connection object referenced by ‘conn’ is automatically + * returned to the pool when ‘conn’ goes out of scope. + */ template <class R> class Pool { public: - /* A function that produces new instances of R on demand. */ + /** + * A function that produces new instances of R on demand. + */ typedef std::function<ref<R>()> Factory; - /* A function that checks whether an instance of R is still - usable. Unusable instances are removed from the pool. */ + /** + * A function that checks whether an instance of R is still + * usable. Unusable instances are removed from the pool. + */ typedef std::function<bool(const ref<R> &)> Validator; private: diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 7d38b059c..af5f8304c 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <memory> #include <exception> @@ -6,8 +7,10 @@ namespace nix { -/* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents null constructions. */ +/** + * A simple non-nullable reference-counted pointer. Actually a wrapper + * around std::shared_ptr that prevents null constructions. + */ template<typename T> class ref { diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 0b997b25a..87d6aa678 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string_view> diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c653db9d0..7476e6f6c 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -415,7 +415,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(std::move(format("%s") % msg)), + .msg = hintformat(fmt("%s", msg)), }; auto havePos = readNum<size_t>(source); assert(havePos == 0); @@ -424,7 +424,7 @@ Error readError(Source & source) havePos = readNum<size_t>(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(std::move(format("%s") % readString(source))) + .hint = hintformat(fmt("%s", readString(source))) }); } return Error(std::move(info)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 7da5b07fd..2cf527023 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <memory> @@ -10,7 +11,9 @@ namespace boost::context { struct stack_context; } namespace nix { -/* Abstract destination of binary data. */ +/** + * Abstract destination of binary data. + */ struct Sink { virtual ~Sink() { } @@ -18,7 +21,9 @@ struct Sink virtual bool good() { return true; } }; -/* Just throws away data. */ +/** + * Just throws away data. + */ struct NullSink : Sink { void operator () (std::string_view data) override @@ -32,8 +37,10 @@ struct FinishSink : virtual Sink }; -/* A buffered abstract sink. Warning: a BufferedSink should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract sink. Warning: a BufferedSink should not be + * used from multiple threads concurrently. + */ struct BufferedSink : virtual Sink { size_t bufSize, bufPos; @@ -50,19 +57,25 @@ struct BufferedSink : virtual Sink }; -/* Abstract source of binary data. */ +/** + * Abstract source of binary data. + */ struct Source { virtual ~Source() { } - /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. - It blocks until all the requested data is available, or throws - an error if it is not going to be available. */ + /** + * Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + * It blocks until all the requested data is available, or throws + * an error if it is not going to be available. + */ void operator () (char * data, size_t len); - /* Store up to ‘len’ in the buffer pointed to by ‘data’, and - return the number of bytes stored. It blocks until at least - one byte is available. */ + /** + * Store up to ‘len’ in the buffer pointed to by ‘data’, and + * return the number of bytes stored. It blocks until at least + * one byte is available. + */ virtual size_t read(char * data, size_t len) = 0; virtual bool good() { return true; } @@ -73,8 +86,10 @@ struct Source }; -/* A buffered abstract source. Warning: a BufferedSource should not be - used from multiple threads concurrently. */ +/** + * A buffered abstract source. Warning: a BufferedSource should not be + * used from multiple threads concurrently. + */ struct BufferedSource : Source { size_t bufSize, bufPosIn, bufPosOut; @@ -88,12 +103,16 @@ struct BufferedSource : Source bool hasData(); protected: - /* Underlying read call, to be overridden. */ + /** + * Underlying read call, to be overridden. + */ virtual size_t readUnbuffered(char * data, size_t len) = 0; }; -/* A sink that writes data to a file descriptor. */ +/** + * A sink that writes data to a file descriptor. + */ struct FdSink : BufferedSink { int fd; @@ -123,7 +142,9 @@ private: }; -/* A source that reads data from a file descriptor. */ +/** + * A source that reads data from a file descriptor. + */ struct FdSource : BufferedSource { int fd; @@ -149,7 +170,9 @@ private: }; -/* A sink that writes data to a string. */ +/** + * A sink that writes data to a string. + */ struct StringSink : Sink { std::string s; @@ -163,7 +186,9 @@ struct StringSink : Sink }; -/* A source that reads data from a string. */ +/** + * A source that reads data from a string. + */ struct StringSource : Source { std::string_view s; @@ -173,7 +198,9 @@ struct StringSource : Source }; -/* A sink that writes all incoming data to two other sinks. */ +/** + * A sink that writes all incoming data to two other sinks. + */ struct TeeSink : Sink { Sink & sink1, & sink2; @@ -186,7 +213,9 @@ struct TeeSink : Sink }; -/* Adapter class of a Source that saves all data read to a sink. */ +/** + * Adapter class of a Source that saves all data read to a sink. + */ struct TeeSource : Source { Source & orig; @@ -201,7 +230,9 @@ struct TeeSource : Source } }; -/* A reader that consumes the original Source until 'size'. */ +/** + * A reader that consumes the original Source until 'size'. + */ struct SizedSource : Source { Source & orig; @@ -219,7 +250,9 @@ struct SizedSource : Source return n; } - /* Consume the original source until no remain data is left to consume. */ + /** + * Consume the original source until no remain data is left to consume. + */ size_t drainAll() { std::vector<char> buf(8192); @@ -232,7 +265,9 @@ struct SizedSource : Source } }; -/* A sink that that just counts the number of bytes given to it */ +/** + * A sink that that just counts the number of bytes given to it + */ struct LengthSink : Sink { uint64_t length = 0; @@ -243,7 +278,9 @@ struct LengthSink : Sink } }; -/* Convert a function into a sink. */ +/** + * Convert a function into a sink. + */ struct LambdaSink : Sink { typedef std::function<void(std::string_view data)> lambda_t; @@ -259,7 +296,9 @@ struct LambdaSink : Sink }; -/* Convert a function into a source. */ +/** + * Convert a function into a source. + */ struct LambdaSource : Source { typedef std::function<size_t(char *, size_t)> lambda_t; @@ -274,8 +313,10 @@ struct LambdaSource : Source } }; -/* Chain two sources together so after the first is exhausted, the second is - used */ +/** + * Chain two sources together so after the first is exhausted, the second is + * used + */ struct ChainSource : Source { Source & source1, & source2; @@ -289,8 +330,10 @@ struct ChainSource : Source std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun); -/* Convert a function that feeds data into a Sink into a Source. The - Source executes the function as a coroutine. */ +/** + * Convert a function that feeds data into a Sink into a Source. The + * Source executes the function as a coroutine. + */ std::unique_ptr<Source> sinkToSource( std::function<void(Sink &)> fun, std::function<void()> eof = []() { @@ -376,7 +419,9 @@ Source & operator >> (Source & in, bool & b) Error readError(Source & source); -/* An adapter that converts a std::basic_istream into a source. */ +/** + * An adapter that converts a std::basic_istream into a source. + */ struct StreamToSourceAdapter : Source { std::shared_ptr<std::basic_istream<char>> istream; @@ -399,13 +444,14 @@ struct StreamToSourceAdapter : Source }; -/* A source that reads a distinct format of concatenated chunks back into its - logical form, in order to guarantee a known state to the original stream, - even in the event of errors. - - Use with FramedSink, which also allows the logical stream to be terminated - in the event of an exception. -*/ +/** + * A source that reads a distinct format of concatenated chunks back into its + * logical form, in order to guarantee a known state to the original stream, + * even in the event of errors. + * + * Use with FramedSink, which also allows the logical stream to be terminated + * in the event of an exception. + */ struct FramedSource : Source { Source & from; @@ -450,11 +496,12 @@ struct FramedSource : Source } }; -/* Write as chunks in the format expected by FramedSource. - - The exception_ptr reference can be used to terminate the stream when you - detect that an error has occurred on the remote end. -*/ +/** + * Write as chunks in the format expected by FramedSource. + * + * The exception_ptr reference can be used to terminate the stream when you + * detect that an error has occurred on the remote end. + */ struct FramedSink : nix::BufferedSink { BufferedSink & to; @@ -487,17 +534,20 @@ struct FramedSink : nix::BufferedSink }; }; -/* Stack allocation strategy for sinkToSource. - Mutable to avoid a boehm gc dependency in libutil. - - boost::context doesn't provide a virtual class, so we define our own. +/** + * Stack allocation strategy for sinkToSource. + * Mutable to avoid a boehm gc dependency in libutil. + * + * boost::context doesn't provide a virtual class, so we define our own. */ struct StackAllocator { virtual boost::context::stack_context allocate() = 0; virtual void deallocate(boost::context::stack_context sctx) = 0; - /* The stack allocator to use in sinkToSource and potentially elsewhere. - It is reassigned by the initGC() method in libexpr. */ + /** + * The stack allocator to use in sinkToSource and potentially elsewhere. + * It is reassigned by the initGC() method in libexpr. + */ static StackAllocator *defaultAllocator; }; diff --git a/src/libutil/split.hh b/src/libutil/split.hh index 87a23b13e..3b9b2b83b 100644 --- a/src/libutil/split.hh +++ b/src/libutil/split.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <optional> #include <string_view> @@ -7,10 +8,12 @@ namespace nix { -// If `separator` is found, we return the portion of the string before the -// separator, and modify the string argument to contain only the part after the -// separator. Otherwise, we return `std::nullopt`, and we leave the argument -// string alone. +/** + * If `separator` is found, we return the portion of the string before the + * separator, and modify the string argument to contain only the part after the + * separator. Otherwise, we return `std::nullopt`, and we leave the argument + * string alone. + */ static inline std::optional<std::string_view> splitPrefixTo(std::string_view & string, char separator) { auto sepInstance = string.find(separator); diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index d54dd8e31..9abf5ee5f 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "comparator.hh" #include "types.hh" @@ -13,7 +14,8 @@ int levenshteinDistance(std::string_view first, std::string_view second); */ class Suggestion { public: - int distance; // The smaller the better + /// The smaller the better + int distance; std::string suggestion; std::string to_string() const; @@ -43,7 +45,9 @@ public: std::ostream & operator<<(std::ostream & str, const Suggestion &); std::ostream & operator<<(std::ostream & str, const Suggestions &); -// Either a value of type `T`, or some suggestions +/** + * Either a value of type `T`, or some suggestions + */ template<typename T> class OrSuggestions { public: diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index e1d591d77..47e4512b1 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <cstdlib> #include <mutex> @@ -7,22 +8,22 @@ namespace nix { -/* This template class ensures synchronized access to a value of type - T. It is used as follows: - - struct Data { int x; ... }; - - Sync<Data> data; - - { - auto data_(data.lock()); - data_->x = 123; - } - - Here, "data" is automatically unlocked when "data_" goes out of - scope. -*/ - +/** + * This template class ensures synchronized access to a value of type + * T. It is used as follows: + * + * struct Data { int x; ... }; + * + * Sync<Data> data; + * + * { + * auto data_(data.lock()); + * data_->x = 123; + * } + * + * Here, "data" is automatically unlocked when "data_" goes out of + * scope. + */ template<class T, class M = std::mutex> class Sync { diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 4d9141fd4..24afb710a 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "serialise.hh" #include <archive.h> @@ -14,7 +17,7 @@ struct TarArchive { TarArchive(const Path & path); - // disable copy constructor + /// disable copy constructor TarArchive(const TarArchive &) = delete; void close(); diff --git a/src/libutil/tests/canon-path.cc b/src/libutil/tests/canon-path.cc index c1c5adadf..fc94ccc3d 100644 --- a/src/libutil/tests/canon-path.cc +++ b/src/libutil/tests/canon-path.cc @@ -107,15 +107,13 @@ namespace nix { } TEST(CanonPath, within) { - { - ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); - ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); - ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); - ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); - ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); - } + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); } TEST(CanonPath, sort) { @@ -127,29 +125,38 @@ namespace nix { } TEST(CanonPath, allowed) { - { - std::set<CanonPath> allowed { - CanonPath("foo/bar"), - CanonPath("foo!"), - CanonPath("xyzzy"), - CanonPath("a/b/c"), - }; - - ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); - ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); - ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); - } + std::set<CanonPath> allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; + + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + + TEST(CanonPath, makeRelative) { + CanonPath d("/foo/bar"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar")), "."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo")), ".."); + ASSERT_EQ(d.makeRelative(CanonPath("/")), "../.."); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy")), "xyzzy"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/bar/xyzzy/bla")), "xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/foo/xyzzy/bla")), "../xyzzy/bla"); + ASSERT_EQ(d.makeRelative(CanonPath("/xyzzy/bla")), "../../xyzzy/bla"); } } diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh index 9e9650e6e..1f9fa59ae 100644 --- a/src/libutil/tests/hash.hh +++ b/src/libutil/tests/hash.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index e0c438b4d..a908631e6 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -302,4 +302,37 @@ namespace nix { ASSERT_EQ(d, s); } + + /* ---------------------------------------------------------------------------- + * percentEncode + * --------------------------------------------------------------------------*/ + + TEST(percentEncode, encodesUrlEncodedString) { + std::string s = percentEncode("==@=="); + std::string d = "%3D%3D%40%3D%3D"; + ASSERT_EQ(d, s); + } + + TEST(percentEncode, keepArgument) { + std::string a = percentEncode("abd / def"); + std::string b = percentEncode("abd / def", "/"); + ASSERT_EQ(a, "abd%20%2F%20def"); + ASSERT_EQ(b, "abd%20/%20def"); + } + + TEST(percentEncode, inverseOfDecode) { + std::string original = "%3D%3D%40%3D%3D"; + std::string once = percentEncode(original); + std::string back = percentDecode(once); + + ASSERT_EQ(back, original); + } + + TEST(percentEncode, trailingPercent) { + std::string s = percentEncode("==@==%"); + std::string d = "%3D%3D%40%3D%3D%25"; + + ASSERT_EQ(d, s); + } + } diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index b22e0d162..14b32279c 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sync.hh" #include "util.hh" @@ -13,8 +14,10 @@ namespace nix { MakeError(ThreadPoolShutDown, Error); -/* A simple thread pool that executes a queue of work items - (lambdas). */ +/** + * A simple thread pool that executes a queue of work items + * (lambdas). + */ class ThreadPool { public: @@ -23,19 +26,30 @@ public: ~ThreadPool(); - // FIXME: use std::packaged_task? + /** + * An individual work item. + * + * \todo use std::packaged_task? + */ typedef std::function<void()> work_t; - /* Enqueue a function to be executed by the thread pool. */ + /** + * Enqueue a function to be executed by the thread pool. + */ void enqueue(const work_t & t); - /* Execute work items until the queue is empty. Note that work - items are allowed to add new items to the queue; this is - handled correctly. Queue processing stops prematurely if any - work item throws an exception. This exception is propagated to - the calling thread. If multiple work items throw an exception - concurrently, only one item is propagated; the others are - printed on stderr and otherwise ignored. */ + /** + * Execute work items until the queue is empty. + * + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. + */ void process(); private: @@ -62,9 +76,11 @@ private: void shutdown(); }; -/* Process in parallel a set of items of type T that have a partial - ordering between them. Thus, any item is only processed after all - its dependencies have been processed. */ +/** + * Process in parallel a set of items of type T that have a partial + * ordering between them. Thus, any item is only processed after all + * its dependencies have been processed. + */ template<typename T> void processGraph( ThreadPool & pool, diff --git a/src/libutil/topo-sort.hh b/src/libutil/topo-sort.hh index 7418be5e0..a52811fbf 100644 --- a/src/libutil/topo-sort.hh +++ b/src/libutil/topo-sort.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 6bcbd7e1d..c86f52175 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" @@ -17,7 +18,9 @@ typedef std::set<std::string> StringSet; typedef std::map<std::string, std::string> StringMap; typedef std::map<std::string, std::string> StringPairs; -/* Paths are just strings. */ +/** + * Paths are just strings. + */ typedef std::string Path; typedef std::string_view PathView; typedef std::list<Path> Paths; @@ -25,15 +28,19 @@ typedef std::set<Path> PathSet; typedef std::vector<std::pair<std::string, std::string>> Headers; -/* Helper class to run code at startup. */ +/** + * Helper class to run code at startup. + */ template<typename T> struct OnStartup { OnStartup(T && t) { t(); } }; -/* Wrap bools to prevent string literals (i.e. 'char *') from being - cast to a bool in Attr. */ +/** + * Wrap bools to prevent string literals (i.e. 'char *') from being + * cast to a bool in Attr. + */ template<typename T> struct Explicit { T t; @@ -45,21 +52,25 @@ struct Explicit { }; -/* This wants to be a little bit like rust's Cow type. - Some parts of the evaluator benefit greatly from being able to reuse - existing allocations for strings, but have to be able to also use - newly allocated storage for values. - - We do not define implicit conversions, even with ref qualifiers, - since those can easily become ambiguous to the reader and can degrade - into copying behaviour we want to avoid. */ +/** + * This wants to be a little bit like rust's Cow type. + * Some parts of the evaluator benefit greatly from being able to reuse + * existing allocations for strings, but have to be able to also use + * newly allocated storage for values. + * + * We do not define implicit conversions, even with ref qualifiers, + * since those can easily become ambiguous to the reader and can degrade + * into copying behaviour we want to avoid. + */ class BackedStringView { private: std::variant<std::string, std::string_view> data; - /* Needed to introduce a temporary since operator-> must return - a pointer. Without this we'd need to store the view object - even when we already own a string. */ + /** + * Needed to introduce a temporary since operator-> must return + * a pointer. Without this we'd need to store the view object + * even when we already own a string. + */ class Ptr { private: std::string_view view; @@ -77,8 +88,10 @@ public: BackedStringView(const BackedStringView &) = delete; BackedStringView & operator=(const BackedStringView &) = delete; - /* We only want move operations defined since the sole purpose of - this type is to avoid copies. */ + /** + * We only want move operations defined since the sole purpose of + * this type is to avoid copies. + */ BackedStringView(BackedStringView && other) = default; BackedStringView & operator=(BackedStringView && other) = default; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index d5e6a2736..98162b0f7 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string> #include <regex> @@ -22,21 +23,22 @@ const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; -// A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check +/// A Git ref (i.e. branch or tag name). +/// \todo check that this is correct. +const static std::string refRegexS = "[a-zA-Z0-9@][a-zA-Z0-9_.\\/@-]*"; extern std::regex refRegex; -// Instead of defining what a good Git Ref is, we define what a bad Git Ref is -// This is because of the definition of a ref in refs.c in https://github.com/git/git -// See tests/fetchGitRefs.sh for the full definition +/// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +/// This is because of the definition of a ref in refs.c in https://github.com/git/git +/// See tests/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; -// A Git revision (a SHA-1 commit hash). +/// A Git revision (a SHA-1 commit hash). const static std::string revRegexS = "[0-9a-fA-F]{40}"; extern std::regex revRegex; -// A ref or revision, or a ref followed by a revision. +/// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 4e43455e1..9e44241ac 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -88,17 +88,22 @@ std::map<std::string, std::string> decodeQuery(const std::string & query) return result; } -std::string percentEncode(std::string_view s) +const static std::string allowedInQuery = ":@/?"; +const static std::string allowedInPath = ":@/"; + +std::string percentEncode(std::string_view s, std::string_view keep) { std::string res; for (auto & c : s) + // unreserved + keep if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') - || strchr("-._~!$&'()*+,;=:@", c)) + || strchr("-._~", c) + || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02x", (unsigned int) c); + res += fmt("%%%02X", (unsigned int) c); return res; } @@ -109,9 +114,9 @@ std::string encodeQuery(const std::map<std::string, std::string> & ss) for (auto & [name, value] : ss) { if (!first) res += '&'; first = false; - res += percentEncode(name); + res += percentEncode(name, allowedInQuery); res += '='; - res += percentEncode(value); + res += percentEncode(value, allowedInQuery); } return res; } @@ -122,7 +127,7 @@ std::string ParsedURL::to_string() const scheme + ":" + (authority ? "//" + *authority : "") - + path + + percentEncode(path, allowedInPath) + (query.empty() ? "" : "?" + encodeQuery(query)) + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 2a9fb34c1..d2413ec0e 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "error.hh" @@ -7,7 +8,8 @@ namespace nix { struct ParsedURL { std::string url; - std::string base; // URL without query/fragment + /// URL without query/fragment + std::string base; std::string scheme; std::optional<std::string> authority; std::string path; @@ -22,12 +24,13 @@ struct ParsedURL MakeError(BadURL, Error); std::string percentDecode(std::string_view in); +std::string percentEncode(std::string_view s, std::string_view keep=""); std::map<std::string, std::string> decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); -/* +/** * Although that’s not really standardized anywhere, an number of tools * use a scheme of the form 'x+y' in urls, where y is the “transport layer” * scheme, and x is the “application layer” scheme. diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 885bae69c..843a10eab 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -54,6 +54,11 @@ std::optional<std::string> getEnv(const std::string & key) return std::string(value); } +std::optional<std::string> getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} std::map<std::string, std::string> getEnv() { @@ -523,7 +528,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, uint64_t & bytesFreed) { - //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -1065,12 +1070,14 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) } +#if __linux__ static int childEntry(void * arg) { auto main = (std::function<void()> *) arg; (*main)(); return 1; } +#endif pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) @@ -1394,14 +1401,14 @@ std::string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { if (WIFEXITED(status)) - return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + return fmt("failed with exit code %1%", WEXITSTATUS(status)); else if (WIFSIGNALED(status)) { int sig = WTERMSIG(status); #if HAVE_STRSIGNAL const char * description = strsignal(sig); - return (format("failed due to signal %1% (%2%)") % sig % description).str(); + return fmt("failed due to signal %1% (%2%)", sig, description); #else - return (format("failed due to signal %1%") % sig).str(); + return fmt("failed due to signal %1%", sig); #endif } else @@ -1470,7 +1477,7 @@ bool shouldANSI() && !getEnv("NO_COLOR").has_value(); } -std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) { std::string t, e; size_t w = 0; @@ -1961,7 +1968,7 @@ std::string showBytes(uint64_t bytes) // FIXME: move to libstore/build -void commonChildInit(Pipe & logPipe) +void commonChildInit() { logger = makeSimpleLogger(); @@ -1975,10 +1982,6 @@ void commonChildInit(Pipe & logPipe) if (setsid() == -1) throw SysError("creating a new session"); - /* Dup the write side of the logger pipe into stderr. */ - if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) - throw SysError("cannot pipe standard error into log file"); - /* Dup stderr to stdout. */ if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) throw SysError("cannot dup stderr into stdout"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b5625ecef..6c2706cc1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "error.hh" @@ -39,6 +40,10 @@ extern const std::string nativeSystem; /* Return an environment variable. */ std::optional<std::string> getEnv(const std::string & key); +/* Return a non empty environment variable. Returns nullopt if the env +variable is set to "" */ +std::optional<std::string> getEnvNonEmpty(const std::string & key); + /* Get the entire environment. */ std::map<std::string, std::string> getEnv(); @@ -446,7 +451,6 @@ template<class C> Strings quoteStrings(const C & c) return res; } - /* Remove trailing whitespace from a string. FIXME: return std::string_view. */ std::string chomp(std::string_view s); @@ -569,7 +573,7 @@ bool shouldANSI(); some escape sequences (such as colour setting) are copied but not included in the character count. Also, tabs are expanded to spaces. */ -std::string filterANSIEscapes(const std::string & s, +std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits<unsigned int>::max()); @@ -700,7 +704,7 @@ typedef std::function<bool(const Path & path)> PathFilter; extern PathFilter defaultPathFilter; /* Common initialisation performed in child processes. */ -void commonChildInit(Pipe & logPipe); +void commonChildInit(); /* Create a Unix domain socket. */ AutoCloseFD createUnixDomainSocket(); diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index 4c91adee6..74f53b7ca 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <iostream> #include <string> diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index da76c2ace..bc7e7eb18 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -219,9 +219,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); } } @@ -440,7 +440,7 @@ static void main_nix_build(int argc, char * * argv) shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); drv = *resolvedDrv; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 338a7d18e..740737ffe 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -168,7 +168,8 @@ static int main_nix_channel(int argc, char ** argv) nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = profilesDir() + "/channels"; + profile = profilesDir() + "/channels"; + createDirs(dirOf(profile)); enum { cNone, diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index e413faffe..3cc57af4e 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -40,7 +40,7 @@ void removeOldGenerations(std::string dir) throw; } if (link.find("link") != std::string::npos) { - printInfo(format("removing old generations of profile %1%") % path); + printInfo("removing old generations of profile %s", path); if (deleteOlderThan != "") deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); else @@ -77,8 +77,7 @@ static int main_nix_collect_garbage(int argc, char * * argv) return true; }); - auto profilesDir = settings.nixStateDir + "/profiles"; - if (removeOld) removeOldGenerations(profilesDir); + if (removeOld) removeOldGenerations(profilesDir()); // Run the actual garbage collector. if (!dryRun) { diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 841d50fd3..7f2bb93b6 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -22,7 +22,7 @@ static int main_nix_copy_closure(int argc, char ** argv) printVersion("nix-copy-closure"); else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { if (*arg != "--gzip") - printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg); + warn("'%1%' is not implemented, falling back to gzip", *arg); gzip = true; } else if (*arg == "--from") toMode = false; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 0daf374de..f076ffdb0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -500,7 +500,7 @@ static bool keep(DrvInfo & drv) static void installDerivations(Globals & globals, const Strings & args, const Path & profile) { - debug(format("installing derivations")); + debug("installing derivations"); /* Get the set of user environment elements to be installed. */ DrvInfos newElems, newElemsTmp; @@ -579,7 +579,7 @@ typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; static void upgradeDerivations(Globals & globals, const Strings & args, UpgradeType upgradeType) { - debug(format("upgrading derivations")); + debug("upgrading derivations"); /* Upgrade works as follows: we take all currently installed derivations, and for any derivation matching any selector, look @@ -768,7 +768,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - debug(format("switching to new user environment")); + debug("switching to new user environment"); Path generation = createGeneration( ref<LocalFSStore>(store2), globals.profile, @@ -1093,7 +1093,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); + //Activity act(*logger, lvlDebug, "outputting query result '%1%'", i.attrPath); if (globals.prebuiltOnly && !validPaths.count(i.queryOutPath()) && @@ -1229,11 +1229,11 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); + attrs2["value"] = fmt("%1%", v->integer); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nFloat) { attrs2["type"] = "float"; - attrs2["value"] = (format("%1%") % v->fpoint).str(); + attrs2["value"] = fmt("%1%", v->fpoint); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nBool) { attrs2["type"] = "bool"; @@ -1337,11 +1337,11 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs for (auto & i : gens) { tm t; if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); - cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") - % i.number - % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday - % t.tm_hour % t.tm_min % t.tm_sec - % (i.number == curGen ? "(current)" : ""); + logger->cout("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||", + i.number, + t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, + t.tm_hour, t.tm_min, t.tm_sec, + i.number == curGen ? "(current)" : ""); } } @@ -1387,6 +1387,8 @@ static int main_nix_env(int argc, char * * argv) { Strings opFlags, opArgs; Operation op = 0; + std::string opName; + bool showHelp = false; RepairFlag repair = NoRepair; std::string file; @@ -1403,11 +1405,11 @@ static int main_nix_env(int argc, char * * argv) try { createDirs(globals.instSource.nixExprPath); replaceSymlink( - fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + defaultChannelsDir(), globals.instSource.nixExprPath + "/channels"); if (getuid() != 0) replaceSymlink( - fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + rootChannelsDir(), globals.instSource.nixExprPath + "/channels_root"); } catch (Error &) { } } @@ -1426,37 +1428,59 @@ static int main_nix_env(int argc, char * * argv) Operation oldOp = op; if (*arg == "--help") - showManPage("nix-env"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--install" || *arg == "-i") + else if (*arg == "--install" || *arg == "-i") { op = opInstall; + opName = "-install"; + } else if (*arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = getArg(*arg, arg, end); - else if (*arg == "--uninstall" || *arg == "-e") + else if (*arg == "--uninstall" || *arg == "-e") { op = opUninstall; - else if (*arg == "--upgrade" || *arg == "-u") + opName = "-uninstall"; + } + else if (*arg == "--upgrade" || *arg == "-u") { op = opUpgrade; - else if (*arg == "--set-flag") + opName = "-upgrade"; + } + else if (*arg == "--set-flag") { op = opSetFlag; - else if (*arg == "--set") + opName = arg->substr(1); + } + else if (*arg == "--set") { op = opSet; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; + opName = "-query"; + } else if (*arg == "--profile" || *arg == "-p") globals.profile = absPath(getArg(*arg, arg, end)); else if (*arg == "--file" || *arg == "-f") file = getArg(*arg, arg, end); - else if (*arg == "--switch-profile" || *arg == "-S") + else if (*arg == "--switch-profile" || *arg == "-S") { op = opSwitchProfile; - else if (*arg == "--switch-generation" || *arg == "-G") + opName = "-switch-profile"; + } + else if (*arg == "--switch-generation" || *arg == "-G") { op = opSwitchGeneration; - else if (*arg == "--rollback") + opName = "-switch-generation"; + } + else if (*arg == "--rollback") { op = opRollback; - else if (*arg == "--list-generations") + opName = arg->substr(1); + } + else if (*arg == "--list-generations") { op = opListGenerations; - else if (*arg == "--delete-generations") + opName = arg->substr(1); + } + else if (*arg == "--delete-generations") { op = opDeleteGenerations; + opName = arg->substr(1); + } else if (*arg == "--dry-run") { printInfo("(dry run; not doing anything)"); globals.dryRun = true; @@ -1485,6 +1509,7 @@ static int main_nix_env(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); + if (showHelp) showManPage("nix-env" + opName); if (!op) throw UsageError("no operation specified"); auto store = openStore(); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index cad7f9c88..745e9e174 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -41,7 +41,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, if (auto drvPath = i.queryDrvPath()) drvsToBuild.push_back({*drvPath}); - debug(format("building user environment dependencies")); + debug("building user environment dependencies"); state.store->buildPaths( toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); @@ -159,7 +159,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, return false; } - debug(format("switching to new user environment")); + debug("switching to new user environment"); Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); switchLink(profile, generation); } diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 10646f713..af45d2d85 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "get-drvs.hh" diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 73b8d06b9..4fd944080 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 425d61e53..439557658 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -57,7 +57,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" << " xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n" - << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='long'/>" << "<key id='name' for='node' attr.name='name' attr.type='string'/>" << "<key id='type' for='node' attr.name='type' attr.type='string'/>" << "<graph id='G' edgedefault='directed'>\n"; diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh index 78be8a367..bd3a4a37c 100644 --- a/src/nix-store/graphml.hh +++ b/src/nix-store/graphml.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3bbefedbe..7035e6a7b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -72,11 +72,13 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true) Derivation drv = store->derivationFromPath(path.path); rootNr++; + /* FIXME: Encode this empty special case explicitly in the type. */ if (path.outputs.empty()) for (auto & i : drv.outputs) path.outputs.insert(i.first); PathSet outputs; for (auto & j : path.outputs) { + /* Match outputs of a store path with outputs of the derivation that produces it. */ DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have an output named '%s'", @@ -141,6 +143,7 @@ static void opRealise(Strings opFlags, Strings opArgs) toDerivedPaths(paths), willBuild, willSubstitute, unknown, downloadSize, narSize); + /* Filter out unknown paths from `paths`. */ if (ignoreUnknown) { std::vector<StorePathWithOutputs> paths2; for (auto & i : paths) @@ -274,17 +277,17 @@ static void printTree(const StorePath & path, static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType - { qDefault, qOutputs, qRequisites, qReferences, qReferrers + { qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; - QueryType query = qDefault; + std::optional<QueryType> query; bool useOutput = false; bool includeOutputs = false; bool forceRealise = false; std::string bindingName; for (auto & i : opFlags) { - QueryType prev = query; + std::optional<QueryType> prev = query; if (i == "--outputs") query = qOutputs; else if (i == "--requisites" || i == "-R") query = qRequisites; else if (i == "--references") query = qReferences; @@ -309,15 +312,15 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; else throw UsageError("unknown flag '%1%'", i); - if (prev != qDefault && prev != query) + if (prev && prev != query) throw UsageError("query type '%1%' conflicts with earlier flag", i); } - if (query == qDefault) query = qOutputs; + if (!query) query = qOutputs; RunPager pager; - switch (query) { + switch (*query) { case qOutputs: { for (auto & i : opArgs) { @@ -457,7 +460,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) /* Print each environment variable in the derivation in a format * that can be sourced by the shell. */ for (auto & i : drv.env) - cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + logger->cout("export %1%; %1%=%2%\n", i.first, shellEscape(i.second)); /* Also output the arguments. This doesn't preserve whitespace in arguments. */ @@ -1020,64 +1023,109 @@ static int main_nix_store(int argc, char * * argv) { Strings opFlags, opArgs; Operation op = 0; + bool readFromStdIn = false; + std::string opName; + bool showHelp = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") - showManPage("nix-store"); + showHelp = true; else if (*arg == "--version") op = opVersion; - else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") + else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") { op = opRealise; - else if (*arg == "--add" || *arg == "-A") + opName = "-realise"; + } + else if (*arg == "--add" || *arg == "-A"){ op = opAdd; - else if (*arg == "--add-fixed") + opName = "-add"; + } + else if (*arg == "--add-fixed") { op = opAddFixed; + opName = arg->substr(1); + } else if (*arg == "--print-fixed-path") op = opPrintFixedPath; - else if (*arg == "--delete") + else if (*arg == "--delete") { op = opDelete; - else if (*arg == "--query" || *arg == "-q") + opName = arg->substr(1); + } + else if (*arg == "--query" || *arg == "-q") { op = opQuery; - else if (*arg == "--print-env") + opName = "-query"; + } + else if (*arg == "--print-env") { op = opPrintEnv; - else if (*arg == "--read-log" || *arg == "-l") + opName = arg->substr(1); + } + else if (*arg == "--read-log" || *arg == "-l") { op = opReadLog; - else if (*arg == "--dump-db") + opName = "-read-log"; + } + else if (*arg == "--dump-db") { op = opDumpDB; - else if (*arg == "--load-db") + opName = arg->substr(1); + } + else if (*arg == "--load-db") { op = opLoadDB; + opName = arg->substr(1); + } else if (*arg == "--register-validity") op = opRegisterValidity; else if (*arg == "--check-validity") op = opCheckValidity; - else if (*arg == "--gc") + else if (*arg == "--gc") { op = opGC; - else if (*arg == "--dump") + opName = arg->substr(1); + } + else if (*arg == "--dump") { op = opDump; - else if (*arg == "--restore") + opName = arg->substr(1); + } + else if (*arg == "--restore") { op = opRestore; - else if (*arg == "--export") + opName = arg->substr(1); + } + else if (*arg == "--export") { op = opExport; - else if (*arg == "--import") + opName = arg->substr(1); + } + else if (*arg == "--import") { op = opImport; + opName = arg->substr(1); + } else if (*arg == "--init") op = opInit; - else if (*arg == "--verify") + else if (*arg == "--verify") { op = opVerify; - else if (*arg == "--verify-path") + opName = arg->substr(1); + } + else if (*arg == "--verify-path") { op = opVerifyPath; - else if (*arg == "--repair-path") + opName = arg->substr(1); + } + else if (*arg == "--repair-path") { op = opRepairPath; - else if (*arg == "--optimise" || *arg == "--optimize") + opName = arg->substr(1); + } + else if (*arg == "--optimise" || *arg == "--optimize") { op = opOptimise; - else if (*arg == "--serve") + opName = "-optimise"; + } + else if (*arg == "--serve") { op = opServe; - else if (*arg == "--generate-binary-cache-key") + opName = arg->substr(1); + } + else if (*arg == "--generate-binary-cache-key") { op = opGenerateBinaryCacheKey; + opName = arg->substr(1); + } else if (*arg == "--add-root") gcRoot = absPath(getArg(*arg, arg, end)); + else if (*arg == "--stdin" && !isatty(STDIN_FILENO)) + readFromStdIn = true; else if (*arg == "--indirect") ; else if (*arg == "--no-output") @@ -1090,12 +1138,20 @@ static int main_nix_store(int argc, char * * argv) else opArgs.push_back(*arg); + if (readFromStdIn && op != opImport && op != opRestore && op != opServe) { + std::string word; + while (std::cin >> word) { + opArgs.emplace_back(std::move(word)); + }; + } + if (oldOp && oldOp != op) throw UsageError("only one operation may be specified"); return true; }); + if (showHelp) showManPage("nix-store" + opName); if (!op) throw UsageError("no operation specified"); if (op != opDump && op != opRestore) /* !!! hack */ diff --git a/src/nix/app.cc b/src/nix/app.cc index 5cd65136f..fd4569bb4 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,5 +1,6 @@ #include "installables.hh" #include "installable-derived-path.hh" +#include "installable-value.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" @@ -40,7 +41,7 @@ std::string resolveString( return rewriteStrings(toResolve, rewrites); } -UnresolvedApp Installable::toApp(EvalState & state) +UnresolvedApp InstallableValue::toApp(EvalState & state) { auto cursor = getCursor(state); auto attrPath = cursor->getAttrPath(); @@ -119,11 +120,11 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store) { auto res = unresolved; - std::vector<std::shared_ptr<Installable>> installableContext; + Installables installableContext; for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared<InstallableDerivedPath>(store, DerivedPath { ctxElt })); + make_ref<InstallableDerivedPath>(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff --git a/src/nix/build.cc b/src/nix/build.cc index 12b22d999..4e133e288 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,4 +1,3 @@ -#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -41,6 +40,29 @@ nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> return res; } +// TODO deduplicate with other code also setting such out links. +static void createOutLinks(const Path& outLink, const std::vector<BuiltPathWithResult>& buildables, LocalFSStore& store2) +{ + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; + std::visit(overloaded { + [&](const BuiltPath::Opaque & bo) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2.addPermRoot(bo.path, absPath(symlink)); + }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2.addPermRoot(output.second, absPath(symlink)); + } + }, + }, buildable.path.raw()); + } +} + struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; @@ -89,7 +111,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile ; } - void run(ref<Store> store) override + void run(ref<Store> store, Installables && installables) override { if (dryRun) { std::vector<DerivedPath> pathsToBuild; @@ -115,35 +137,18 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) - for (const auto & [_i, buildable] : enumerate(buildables)) { - auto i = _i; - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - store2->addPermRoot(bo.path, absPath(symlink)); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink)); - } - }, - }, buildable.path.raw()); - } + createOutLinks(outLink, buildables, *store2); if (printOutputPaths) { stopProgressBar(); for (auto & buildable : buildables) { std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { - std::cout << store->printStorePath(bo.path) << std::endl; + logger->cout(store->printStorePath(bo.path)); }, [&](const BuiltPath::Built & bfd) { for (auto & output : bfd.outputs) { - std::cout << store->printStorePath(output.second) << std::endl; + logger->cout(store->printStorePath(output.second)); } }, }, buildable.path.raw()); diff --git a/src/nix/build.md b/src/nix/build.md index 6a79f308c..ee414dc86 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -82,7 +82,7 @@ R""( # Description -`nix build` builds the specified *installables*. Installables that +`nix build` builds the specified *installables*. [Installables](./nix.md#installables) that resolve to derivations are built (or substituted if possible). Store path installables are substituted. diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 6ae9460f6..57c355f0c 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,13 +1,15 @@ -#include "command.hh" +#include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "fs-accessor.hh" +#include "eval-inline.hh" using namespace nix; -struct CmdBundle : InstallableCommand +struct CmdBundle : InstallableValueCommand { std::string bundler = "github:NixOS/bundlers"; std::optional<Path> outLink; @@ -69,7 +71,7 @@ struct CmdBundle : InstallableCommand return res; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { auto evalState = getEvalState(); diff --git a/src/nix/bundle.md b/src/nix/bundle.md index a18161a3c..89458aaaa 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -29,7 +29,7 @@ R""( # Description -`nix bundle`, by default, packs the closure of the *installable* into a single +`nix bundle`, by default, packs the closure of the [*installable*](./nix.md#installables) into a single self-extracting executable. See the [`bundlers` homepage](https://github.com/NixOS/bundlers) for more details. diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 6420a0f79..60aa66ce0 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -17,7 +17,7 @@ struct MixCat : virtual Args if (st.type != FSAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); - std::cout << accessor->readFile(path); + writeFull(STDOUT_FILENO, accessor->readFile(path)); } }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 8730a9a5c..151d28277 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -10,8 +10,6 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand SubstituteFlag substitute = NoSubstitute; - using BuiltPathsCommand::run; - CmdCopy() : BuiltPathsCommand(true) { diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index a22bccba1..7e4a7ba86 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -249,9 +249,9 @@ static void daemonLoop() if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); - printInfo(format((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) - % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") - % (peer.uidKnown ? user : "<unknown>")); + printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), + peer.pidKnown ? std::to_string(peer.pid) : "<unknown>", + peer.uidKnown ? user : "<unknown>"); // Fork a child to handle the connection. ProcessOptions options; diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc deleted file mode 100644 index 1dd384c0e..000000000 --- a/src/nix/describe-stores.cc +++ /dev/null @@ -1,44 +0,0 @@ -#include "command.hh" -#include "common-args.hh" -#include "shared.hh" -#include "store-api.hh" - -#include <nlohmann/json.hpp> - -using namespace nix; - -struct CmdDescribeStores : Command, MixJSON -{ - std::string description() override - { - return "show registered store types and their available options"; - } - - Category category() override { return catUtility; } - - void run() override - { - auto res = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); - res[storeName] = storeConfig->toJSON(); - } - if (json) { - std::cout << res; - } else { - for (auto & [storeName, storeConfig] : res.items()) { - std::cout << "## " << storeName << std::endl << std::endl; - for (auto & [optionName, optionDesc] : storeConfig.items()) { - std::cout << "### " << optionName << std::endl << std::endl; - std::cout << optionDesc["description"].get<std::string>() << std::endl; - std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl; - if (!optionDesc["aliases"].empty()) - std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl; - } - } - } - } -}; - -static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 16bbd8613..9e2dcff61 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,5 +1,6 @@ #include "eval.hh" -#include "command.hh" +#include "installable-flake.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -207,7 +208,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore drv.name += "-env"; drv.env.emplace("name", drv.name); drv.inputSrcs.insert(std::move(getEnvShPath)); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { output.second = DerivationOutput::Deferred {}, drv.env[output.first] = hashPlaceholder(output.first); @@ -251,7 +252,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore throw Error("get-env.sh failed to produce an environment"); } -struct Common : InstallableCommand, MixProfile +struct Common : InstallableValueCommand, MixProfile { std::set<std::string> ignoreVars{ "BASHOPTS", @@ -312,7 +313,7 @@ struct Common : InstallableCommand, MixProfile buildEnvironment.toBash(out, ignoreVars); for (auto & var : savedVars) - out << fmt("%s=\"$%s:$nix_saved_%s\"\n", var, var, var); + out << fmt("%s=\"$%s${nix_saved_%s:+:$nix_saved_%s}\"\n", var, var, var, var); out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) @@ -373,7 +374,7 @@ struct Common : InstallableCommand, MixProfile return res; } - StorePath getShellOutPath(ref<Store> store) + StorePath getShellOutPath(ref<Store> store, ref<InstallableValue> installable) { auto path = installable->getStorePath(); if (path && hasSuffix(path->to_string(), "-env")) @@ -391,9 +392,10 @@ struct Common : InstallableCommand, MixProfile } } - std::pair<BuildEnvironment, std::string> getBuildEnvironment(ref<Store> store) + std::pair<BuildEnvironment, std::string> + getBuildEnvironment(ref<Store> store, ref<InstallableValue> installable) { - auto shellOutPath = getShellOutPath(store); + auto shellOutPath = getShellOutPath(store, installable); auto strPath = store->printStorePath(shellOutPath); @@ -479,9 +481,9 @@ struct CmdDevelop : Common, MixEnvironment ; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { - auto [buildEnvironment, gcroot] = getBuildEnvironment(store); + auto [buildEnvironment, gcroot] = getBuildEnvironment(store, installable); auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); @@ -536,7 +538,7 @@ struct CmdDevelop : Common, MixEnvironment nixpkgsLockFlags.inputOverrides = {}; nixpkgsLockFlags.inputUpdates = {}; - auto bashInstallable = std::make_shared<InstallableFlake>( + auto bashInstallable = make_ref<InstallableFlake>( this, state, installable->nixpkgsFlakeRef(), @@ -572,7 +574,7 @@ struct CmdDevelop : Common, MixEnvironment // Need to chdir since phases assume in flake directory if (phase) { // chdir if installable is a flake of type git+file or path - auto installableFlake = std::dynamic_pointer_cast<InstallableFlake>(installable); + auto installableFlake = installable.dynamic_pointer_cast<InstallableFlake>(); if (installableFlake) { auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); if (sourcePath) { @@ -603,9 +605,9 @@ struct CmdPrintDevEnv : Common, MixJSON Category category() override { return catUtility; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { - auto buildEnvironment = getBuildEnvironment(store).first; + auto buildEnvironment = getBuildEnvironment(store, installable).first; stopProgressBar(); diff --git a/src/nix/develop.md b/src/nix/develop.md index 4e8542d1b..c49b39669 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -76,7 +76,7 @@ R""( `nix develop` starts a `bash` shell that provides an interactive build environment nearly identical to what Nix would use to build -*installable*. Inside this shell, environment variables and shell +[*installable*](./nix.md#installables). Inside this shell, environment variables and shell functions are set up so that you can interactively and incrementally build your package. diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 3489cc132..c7c37b66f 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -97,7 +97,7 @@ void printClosureDiff( items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); if (showDelta) items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); - std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items)); + logger->cout("%s%s: %s", indent, name, concatStringsSep(", ", items)); } } } diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index ea87e3d87..354b03cf6 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -18,7 +18,7 @@ std::string formatProtocol(unsigned int proto) if (proto) { auto major = GET_PROTOCOL_MAJOR(proto) >> 8; auto minor = GET_PROTOCOL_MINOR(proto); - return (format("%1%.%2%") % major % minor).str(); + return fmt("%1%.%2%", major, minor); } return "unknown"; } @@ -39,6 +39,14 @@ struct CmdDoctor : StoreCommand { bool success = true; + /** + * This command is stable before the others + */ + std::optional<ExperimentalFeature> experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "check your system for potential problems and print a PASS or FAIL for each check"; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 76a134b1f..66629fab0 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,14 +1,15 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "editor-for.hh" #include <unistd.h> using namespace nix; -struct CmdEdit : InstallableCommand +struct CmdEdit : InstallableValueCommand { std::string description() override { @@ -24,7 +25,7 @@ struct CmdEdit : InstallableCommand Category category() override { return catSecondary; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { auto state = getEvalState(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ccee074e9..43db5150c 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -1,4 +1,4 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -11,13 +11,13 @@ using namespace nix; -struct CmdEval : MixJSON, InstallableCommand +struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional<std::string> apply; std::optional<Path> writeTo; - CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) + CmdEval() : InstallableValueCommand() { addFlag({ .longName = "raw", @@ -54,7 +54,7 @@ struct CmdEval : MixJSON, InstallableCommand Category category() override { return catSecondary; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { if (raw && json) throw UsageError("--raw and --json are mutually exclusive"); @@ -112,11 +112,11 @@ struct CmdEval : MixJSON, InstallableCommand else if (raw) { stopProgressBar(); - std::cout << *state->coerceToString(noPos, *v, context, "while generating the eval command output"); + writeFull(STDOUT_FILENO, *state->coerceToString(noPos, *v, context, "while generating the eval command output")); } else if (json) { - std::cout << printValueAsJSON(*state, true, *v, pos, context, false).dump() << std::endl; + logger->cout("%s", printValueAsJSON(*state, true, *v, pos, context, false)); } else { diff --git a/src/nix/eval.md b/src/nix/eval.md index 61334cde1..3b510737a 100644 --- a/src/nix/eval.md +++ b/src/nix/eval.md @@ -50,7 +50,7 @@ R""( # Description -This command evaluates the Nix expression *installable* and prints the +This command evaluates the given Nix expression and prints the result on standard output. # Output format diff --git a/src/nix/flake.cc b/src/nix/flake.cc index c025bc7a6..cd4ee5921 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "eval.hh" @@ -951,7 +952,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; - std::cout << jsonRoot.dump() << std::endl; + logger->cout("%s", jsonRoot); } else { traverse(*flake.lockFile.root); } @@ -1025,36 +1026,43 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto visitor2 = visitor.getAttr(attrName); - if ((attrPathS[0] == "apps" - || attrPathS[0] == "checks" - || attrPathS[0] == "devShells" - || attrPathS[0] == "legacyPackages" - || attrPathS[0] == "packages") - && (attrPathS.size() == 1 || attrPathS.size() == 2)) { - for (const auto &subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; + try { + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } } + return false; } - return false; - } - if ((attrPathS.size() == 1) - && (attrPathS[0] == "formatter" - || attrPathS[0] == "nixosConfigurations" - || attrPathS[0] == "nixosModules" - || attrPathS[0] == "overlays" - )) { - for (const auto &subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } } + return false; } - return false; - } - // If we don't recognize it, it's probably content - return true; + // If we don't recognize it, it's probably content + return true; + } catch (EvalError & e) { + // Some attrs may contain errors, eg. legacyPackages of + // nixpkgs. We still want to recurse into it, instead of + // skipping it at all. + return true; + } }; std::function<nlohmann::json( @@ -1327,8 +1335,7 @@ struct CmdFlake : NixMultiCommand { if (!command) throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature(Xp::Flakes); - command->second->prepare(); + experimentalFeatureSettings.require(Xp::Flakes); command->second->run(); } }; diff --git a/src/nix/flake.md b/src/nix/flake.md index 810e9ebea..d70f34eeb 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -54,7 +54,7 @@ output attribute). They are also allowed in the `inputs` attribute of a flake, e.g. ```nix -inputs.nixpkgs.url = github:NixOS/nixpkgs; +inputs.nixpkgs.url = "github:NixOS/nixpkgs"; ``` is equivalent to @@ -221,11 +221,46 @@ Currently the `type` attribute can be one of the following: commit hash (`rev`). Note that unlike Git, GitHub allows fetching by commit hash without specifying a branch or tag. + You can also specify `host` as a parameter, to point to a custom GitHub + Enterprise server. + Some examples: * `github:edolstra/dwarffs` * `github:edolstra/dwarffs/unstable` * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + * `github:internal/project?host=company-github.example.org` + +* `gitlab`: Similar to `github`, is a more efficient way to fetch + GitLab repositories. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + Like `github`, these are downloaded as tarball archives. + + The URL syntax for `gitlab` flakes is: + + `gitlab:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?` + + `<rev-or-ref>` works the same as `github`. Either a branch or tag name + (`ref`), or a commit hash (`rev`) can be specified. + + Since GitLab allows for self-hosting, you can specify `host` as + a parameter, to point to any instances other than `gitlab.com`. + + Some examples: + + * `gitlab:veloren/veloren` + * `gitlab:veloren/veloren/master` + * `gitlab:veloren/veloren/80a4d7f13492d916e47d6195be23acae8001985a` + * `gitlab:openldap/openldap?host=git.openldap.org` + + When accessing a project in a (nested) subgroup, make sure to URL-encode any + slashes, i.e. replace `/` with `%2F`: + + * `gitlab:veloren%2Fdev/rfcs` * `sourcehut`: Similar to `github`, is a more efficient way to fetch SourceHut repositories. The following attributes are required: @@ -275,14 +310,14 @@ Currently the `type` attribute can be one of the following: # Flake format As an example, here is a simple `flake.nix` that depends on the -Nixpkgs flake and provides a single package (i.e. an installable -derivation): +Nixpkgs flake and provides a single package (i.e. an +[installable](./nix.md#installables) derivation): ```nix { description = "A flake for building Hello World"; - inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-20.03"; outputs = { self, nixpkgs }: { @@ -317,6 +352,8 @@ The following attributes are supported in `flake.nix`: also contains some metadata about the inputs. These are: * `outPath`: The path in the Nix store of the flake's source tree. + This way, the attribute set can be passed to `import` as if it was a path, + as in the example above (`import nixpkgs`). * `rev`: The commit hash of the flake's repository, if applicable. @@ -374,7 +411,7 @@ inputs.nixpkgs = { Alternatively, you can use the URL-like syntax: ```nix -inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.import-cargo.url = "github:edolstra/import-cargo"; inputs.nixpkgs.url = "nixpkgs"; ``` diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 6f6a4a632..c85eacded 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-value.hh" #include "run.hh" using namespace nix; @@ -31,8 +32,9 @@ struct CmdFmt : SourceExprCommand { auto evalState = getEvalState(); auto evalStore = getEvalStore(); - auto installable = parseInstallable(store, "."); - auto app = installable->toApp(*evalState).resolve(evalStore, store); + auto installable_ = parseInstallable(store, "."); + auto & installable = InstallableValue::require(*installable_); + auto app = installable.toApp(*evalState).resolve(evalStore, store); Strings programArgs{app.program}; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 60d9593a7..9feca9345 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -151,7 +151,6 @@ struct CmdHash : NixMultiCommand { if (!command) throw UsageError("'nix hash' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; @@ -161,11 +160,11 @@ static auto rCmdHash = registerCommand<CmdHash>("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { - HashType ht = htMD5; + std::optional<HashType> ht; bool flat = false; - bool base32 = false; + Base base = Base16; bool truncate = false; - enum { opHash, opTo32, opTo16 } op = opHash; + enum { opHash, opTo } op = opHash; std::vector<std::string> ss; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -174,14 +173,31 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base32") base32 = true; + else if (*arg == "--base16") base = Base16; + else if (*arg == "--base32") base = Base32; + else if (*arg == "--base64") base = Base64; + else if (*arg == "--sri") base = SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); ht = parseHashType(s); } - else if (*arg == "--to-base16") op = opTo16; - else if (*arg == "--to-base32") op = opTo32; + else if (*arg == "--to-base16") { + op = opTo; + base = Base16; + } + else if (*arg == "--to-base32") { + op = opTo; + base = Base32; + } + else if (*arg == "--to-base64") { + op = opTo; + base = Base64; + } + else if (*arg == "--to-sri") { + op = opTo; + base = SRI; + } else if (*arg != "" && arg->at(0) == '-') return false; else @@ -191,17 +207,18 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - cmd.ht = ht; - cmd.base = base32 ? Base32 : Base16; + if (!ht.has_value()) ht = htMD5; + cmd.ht = ht.value(); + cmd.base = base; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(op == opTo32 ? Base32 : Base16); + CmdToBase cmd(base); cmd.args = ss; - cmd.ht = ht; + if (ht.has_value()) cmd.ht = ht; cmd.run(); } diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md new file mode 100644 index 000000000..47ba9b94d --- /dev/null +++ b/src/nix/help-stores.md @@ -0,0 +1,46 @@ +R"( + +Nix supports different types of stores. These are described below. + +## Store URL format + +Stores are specified using a URL-like syntax. For example, the command + +```console +# nix path-info --store https://cache.nixos.org/ --json \ + /nix/store/a7gvj343m05j2s32xcnwr35v31ynlypr-coreutils-9.1 +``` + +fetches information about a store path in the HTTP binary cache +located at https://cache.nixos.org/, which is a type of store. + +Store URLs can specify **store settings** using URL query strings, +i.e. by appending `?name1=value1&name2=value2&...` to the URL. For +instance, + +``` +--store ssh://machine.example.org?ssh-key=/path/to/my/key +``` + +tells Nix to access the store on a remote machine via the SSH +protocol, using `/path/to/my/key` as the SSH private key. The +supported settings for each store type are documented below. + +The special store URL `auto` causes Nix to automatically select a +store as follows: + +* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` + is writable by the current user. + +* Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect + to the Nix daemon listening on that socket](#local-daemon-store). + +* Otherwise, on Linux only, use the [local chroot store](#local-store) + `~/.local/share/nix/root`, which will be created automatically if it + does not exist. + +* Otherwise, use the [local store](#local-store) `/nix/store`. + +@stores@ + +)" diff --git a/src/nix/log.cc b/src/nix/log.cc index a0598ca13..aaf829764 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -23,7 +23,7 @@ struct CmdLog : InstallableCommand Category category() override { return catSecondary; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<Installable> installable) override { settings.readOnlyMode = true; @@ -53,7 +53,7 @@ struct CmdLog : InstallableCommand if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); - std::cout << *log; + writeFull(STDOUT_FILENO, *log); return; } diff --git a/src/nix/log.md b/src/nix/log.md index 1c76226a3..01e9801df 100644 --- a/src/nix/log.md +++ b/src/nix/log.md @@ -22,8 +22,7 @@ R""( # Description -This command prints the log of a previous build of the derivation -*installable* on standard output. +This command prints the log of a previous build of the [*installable*](./nix.md#installables) on standard output. Nix looks for build logs in two places: diff --git a/src/nix/ls.cc b/src/nix/ls.cc index e964b01b3..c990a303c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -93,7 +93,7 @@ struct MixLs : virtual Args, MixJSON if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - std::cout << listNar(accessor, path, recursive); + logger->cout("%s", listNar(accessor, path, recursive)); } else listText(accessor); } diff --git a/src/nix/main.cc b/src/nix/main.cc index d3d2f5b16..4d4164333 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -54,17 +54,17 @@ static bool haveInternet() std::string programPath; -struct HelpRequested { }; - struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool useNet = true; bool refresh = false; + bool helpRequested = false; bool showVersion = false; NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); + categories[catHelp] = "Help commands"; categories[Command::catDefault] = "Main commands"; categories[catSecondary] = "Infrequently used commands"; categories[catUtility] = "Utility/scripting commands"; @@ -74,7 +74,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "help", .description = "Show usage information.", .category = miscCategory, - .handler = {[&]() { throw HelpRequested(); }}, + .handler = {[this]() { this->helpRequested = true; }}, }); addFlag({ @@ -83,6 +83,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -98,6 +99,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, + .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -105,6 +107,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, + .experimentalFeature = Xp::NixCommand, }); } @@ -164,11 +167,29 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { commands = RegisterCommand::getCommandsFor({}); } + + std::string dumpCli() + { + auto res = nlohmann::json::object(); + + res["args"] = toJSON(); + + auto stores = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + stores[storeName]["doc"] = storeConfig->doc(); + stores[storeName]["settings"] = storeConfig->toJSON(); + } + res["stores"] = std::move(stores); + + return res.dump(); + } }; /* Render the help for the specified subcommand to stdout using lowdown. */ -static void showHelp(std::vector<std::string> subcommand, MultiCommand & toplevel) +static void showHelp(std::vector<std::string> subcommand, NixArgs & toplevel) { auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); @@ -189,11 +210,11 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve , "/"), *vUtils); - auto attrs = state.buildBindings(16); - attrs.alloc("toplevel").mkString(toplevel.toJSON().dump()); + auto vDump = state.allocValue(); + vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, state.allocValue()->mkAttrs(attrs), *vRes, noPos); + state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) @@ -205,6 +226,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve std::cout << renderMarkdownToTerminal(markdown) << "\n"; } +static NixArgs & getNixArgs(Command & cmd) +{ + assert(cmd.parent); + MultiCommand * toplevel = cmd.parent; + while (toplevel->parent) toplevel = toplevel->parent; + return dynamic_cast<NixArgs &>(*toplevel); +} + struct CmdHelp : Command { std::vector<std::string> subcommand; @@ -229,17 +258,43 @@ struct CmdHelp : Command ; } + Category category() override { return catHelp; } + void run() override { assert(parent); MultiCommand * toplevel = parent; while (toplevel->parent) toplevel = toplevel->parent; - showHelp(subcommand, *toplevel); + showHelp(subcommand, getNixArgs(*this)); } }; static auto rCmdHelp = registerCommand<CmdHelp>("help"); +struct CmdHelpStores : Command +{ + std::string description() override + { + return "show help about store types and their settings"; + } + + std::string doc() override + { + return + #include "help-stores.md" + ; + } + + Category category() override { return catHelp; } + + void run() override + { + showHelp({"help-stores"}, getNixArgs(*this)); + } +}; + +static auto rCmdHelpStores = registerCommand<CmdHelpStores>("help-stores"); + void mainWrapped(int argc, char * * argv) { savedArgv = argv; @@ -291,13 +346,16 @@ void mainWrapped(int argc, char * * argv) NixArgs args; - if (argc == 2 && std::string(argv[1]) == "__dump-args") { - std::cout << args.toJSON().dump() << "\n"; + if (argc == 2 && std::string(argv[1]) == "__dump-cli") { + logger->cout(args.dumpCli()); return; } if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { - settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure}; + experimentalFeatureSettings.experimentalFeatures = { + Xp::Flakes, + Xp::FetchClosure, + }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); @@ -312,7 +370,7 @@ void mainWrapped(int argc, char * * argv) b["doc"] = trim(stripIndentation(primOp->doc)); res[state.symbols[builtin.name]] = std::move(b); } - std::cout << res.dump() << "\n"; + logger->cout("%s", res); return; } @@ -321,20 +379,24 @@ void mainWrapped(int argc, char * * argv) if (completions) { switch (completionType) { case ctNormal: - std::cout << "normal\n"; break; + logger->cout("normal"); break; case ctFilenames: - std::cout << "filenames\n"; break; + logger->cout("filenames"); break; case ctAttrs: - std::cout << "attrs\n"; break; + logger->cout("attrs"); break; } for (auto & s : *completions) - std::cout << s.completion << "\t" << trim(s.description) << "\n"; + logger->cout(s.completion + "\t" + trim(s.description)); } }); try { args.parseCmdline(argvToStrings(argc, argv)); - } catch (HelpRequested &) { + } catch (UsageError &) { + if (!args.helpRequested && !completions) throw; + } + + if (args.helpRequested) { std::vector<std::string> subcommand; MultiCommand * command = &args; while (command) { @@ -346,8 +408,6 @@ void mainWrapped(int argc, char * * argv) } showHelp(subcommand, args); return; - } catch (UsageError &) { - if (!completions) throw; } if (completions) { @@ -363,10 +423,8 @@ void mainWrapped(int argc, char * * argv) if (!args.command) throw UsageError("no subcommand specified"); - if (args.command->first != "repl" - && args.command->first != "doctor" - && args.command->first != "upgrade-nix") - settings.requireExperimentalFeature(Xp::NixCommand); + experimentalFeatureSettings.require( + args.command->second->experimentalFeature()); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); @@ -394,7 +452,6 @@ void mainWrapped(int argc, char * * argv) if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { evalSettings.pureEval = false; } - args.command->second->prepare(); args.command->second->run(); } diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index d86b90fc7..d9c988a9f 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -28,7 +28,6 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, ; } - using StorePathsCommand::run; void run(ref<Store> srcStore, StorePaths && storePaths) override { auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); @@ -45,7 +44,7 @@ struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, } auto json = json::object(); json["rewrites"] = jsonRewrites; - std::cout << json.dump(); + logger->cout("%s", json); } else { for (auto & path : storePaths) { auto i = remappings.find(path); diff --git a/src/nix/make-content-addressed.md b/src/nix/make-content-addressed.md index 32eecc880..b1f7da525 100644 --- a/src/nix/make-content-addressed.md +++ b/src/nix/make-content-addressed.md @@ -35,7 +35,9 @@ R""( # Description This command converts the closure of the store paths specified by -*installables* to content-addressed form. Nix store paths are usually +[*installables*](./nix.md#installables) to content-addressed form. + +Nix store paths are usually *input-addressed*, meaning that the hash part of the store path is computed from the contents of the derivation (i.e., the build-time dependency graph). Input-addressed paths need to be signed by a diff --git a/src/nix/nar.cc b/src/nix/nar.cc index dbb043d9b..9815410cf 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -25,7 +25,6 @@ struct CmdNar : NixMultiCommand { if (!command) throw UsageError("'nix nar' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff --git a/src/nix/nix.md b/src/nix/nix.md index db60c59ff..e1865b31c 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,102 +48,112 @@ manual](https://nixos.org/manual/nix/stable/). # Installables -Many `nix` subcommands operate on one or more *installables*. These are -command line arguments that represent something that can be built in -the Nix store. Here are the recognised types of installables: - -* **Flake output attributes**: `nixpkgs#hello` - - These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a - flake reference and *attrpath* is an optional attribute path. For - more information on flakes, see [the `nix flake` manual - page](./nix3-flake.md). Flake references are most commonly a flake - identifier in the flake registry (e.g. `nixpkgs`), or a raw path - (e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL - (e.g. `github:nixos/nixpkgs` or `path:.`) - - When the flake reference is a raw path (a path without any URL - scheme), it is interpreted as a `path:` or `git+file:` url in the following - way: - - - If the path is within a Git repository, then the url will be of the form - `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` - where `GIT_REPO_ROOT` is the path to the root of the git repository, - and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory - root) of the closest parent of the given path that contains a `flake.nix` within - the git repository. - If no such directory exists, then Nix will error-out. - - Note that the search will only include files indexed by git. In particular, files - which are matched by `.gitignore` or have never been `git add`-ed will not be - available in the flake. If this is undesirable, specify `path:<directory>` explicitly; - - For example, if `/foo/bar` is a git repository with the following structure: - ``` - . - └── baz - ├── blah - │ └── file.txt - └── flake.nix - ``` +Many `nix` subcommands operate on one or more *installables*. +These are command line arguments that represent something that can be realised in the Nix store. - Then `/foo/bar/baz/blah` will resolve to `git+file:///foo/bar?dir=baz` +The following types of installable are supported by most commands: - - If the supplied path is not a git repository, then the url will have the form - `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent - of the supplied path that contains a `flake.nix` file (within the same file-system). - If no such directory exists, then Nix will error-out. - - For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to - `path:/foo/bar` +- [Flake output attribute](#flake-output-attribute) +- [Store path](#store-path) +- [Nix file](#nix-file), optionally qualified by an attribute path +- [Nix expression](#nix-expression), optionally qualified by an attribute path - If *attrpath* is omitted, Nix tries some default values; for most - subcommands, the default is `packages.`*system*`.default` - (e.g. `packages.x86_64-linux.default`), but some subcommands have - other defaults. If *attrpath* *is* specified, *attrpath* is - interpreted as relative to one or more prefixes; for most - subcommands, these are `packages.`*system*, - `legacyPackages.*system*` and the empty prefix. Thus, on - `x86_64-linux` `nix build nixpkgs#hello` will try to build the - attributes `packages.x86_64-linux.hello`, - `legacyPackages.x86_64-linux.hello` and `hello`. +For most commands, if no installable is specified, `.` as assumed. +That is, Nix will operate on the default flake output attribute of the flake in the current directory. -* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` +### Flake output attribute - These are paths inside the Nix store, or symlinks that resolve to a - path in the Nix store. +Example: `nixpkgs#hello` -* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` +These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a +[flake reference](./nix3-flake.md#flake-references) and *attrpath* is an optional attribute path. For +more information on flakes, see [the `nix flake` manual +page](./nix3-flake.md). Flake references are most commonly a flake +identifier in the flake registry (e.g. `nixpkgs`), or a raw path +(e.g. `/path/to/my-flake` or `.` or `../foo`), or a full URL +(e.g. `github:nixos/nixpkgs` or `path:.`) - By default, if you pass a [store derivation] path to a `nix` subcommand, the command will operate on the [output path]s of the derivation. +When the flake reference is a raw path (a path without any URL +scheme), it is interpreted as a `path:` or `git+file:` url in the following +way: - [output path]: ../../glossary.md#gloss-output-path +- If the path is within a Git repository, then the url will be of the form + `git+file://[GIT_REPO_ROOT]?dir=[RELATIVE_FLAKE_DIR_PATH]` + where `GIT_REPO_ROOT` is the path to the root of the git repository, + and `RELATIVE_FLAKE_DIR_PATH` is the path (relative to the directory + root) of the closest parent of the given path that contains a `flake.nix` within + the git repository. + If no such directory exists, then Nix will error-out. - For example, `nix path-info` prints information about the output paths: + Note that the search will only include files indexed by git. In particular, files + which are matched by `.gitignore` or have never been `git add`-ed will not be + available in the flake. If this is undesirable, specify `path:<directory>` explicitly; - ```console - # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv - [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] + For example, if `/foo/bar` is a git repository with the following structure: + ``` + . + └── baz + ├── blah + │ └── file.txt + └── flake.nix ``` - If you want to operate on the store derivation itself, pass the - `--derivation` flag. + Then `/foo/bar/baz/blah` will resolve to `git+file:///foo/bar?dir=baz` + +- If the supplied path is not a git repository, then the url will have the form + `path:FLAKE_DIR_PATH` where `FLAKE_DIR_PATH` is the closest parent + of the supplied path that contains a `flake.nix` file (within the same file-system). + If no such directory exists, then Nix will error-out. + + For example, if `/foo/bar/flake.nix` exists, then `/foo/bar/baz/` will resolve to + `path:/foo/bar` + +If *attrpath* is omitted, Nix tries some default values; for most +subcommands, the default is `packages.`*system*`.default` +(e.g. `packages.x86_64-linux.default`), but some subcommands have +other defaults. If *attrpath* *is* specified, *attrpath* is +interpreted as relative to one or more prefixes; for most +subcommands, these are `packages.`*system*, +`legacyPackages.*system*` and the empty prefix. Thus, on +`x86_64-linux` `nix build nixpkgs#hello` will try to build the +attributes `packages.x86_64-linux.hello`, +`legacyPackages.x86_64-linux.hello` and `hello`. + +### Store path + +Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` + +These are paths inside the Nix store, or symlinks that resolve to a path in the Nix store. -* **Nix attributes**: `--file /path/to/nixpkgs hello` +A [store derivation] is also addressed by store path. - When the `-f` / `--file` *path* option is given, installables are - interpreted as attribute paths referencing a value returned by - evaluating the Nix file *path*. +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` -* **Nix expressions**: `--expr '(import <nixpkgs> {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. +If you want to refer to an output path of that store derivation, add the output name preceded by a caret (`^`). - When the `--expr` option is given, all installables are interpreted - as Nix expressions. You may need to specify `--impure` if the - expression references impure inputs (such as `<nixpkgs>`). +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv^out` -For most commands, if no installable is specified, the default is `.`, -i.e. Nix will operate on the default flake output attribute of the -flake in the current directory. +All outputs can be referred to at once with the special syntax `^*`. + +Example: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv^*` + +### Nix file + +Example: `--file /path/to/nixpkgs hello` + +When the option `-f` / `--file` *path* \[*attrpath*...\] is given, installables are interpreted as the value of the expression in the Nix file at *path*. +If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. +The Nix expression in that file, or any selected attribute, must evaluate to a derivation. + +### Nix expression + +Example: `--expr 'import <nixpkgs> {}' hello` + +When the option `--expr` *expression* \[*attrpath*...\] is given, installables are interpreted as the value of the of the Nix expression. +If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. +The Nix expression, or any selected attribute, must evaluate to a derivation. + +You may need to specify `--impure` if the expression references impure inputs (such as `<nixpkgs>`). ## Derivation output selection @@ -210,8 +220,7 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. - -TODO: list store types, options +Most `nix` subcommands operate on a *Nix store*. These are documented +in [`nix help-stores`](./nix3-help-stores.md). )"" diff --git a/src/nix/path-info.md b/src/nix/path-info.md index b30898ac0..6ad23a02e 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -80,7 +80,7 @@ R""( # Description This command shows information about the store paths produced by -*installables*, or about all paths in the store if you pass `--all`. +[*installables*](./nix.md#installables), or about all paths in the store if you pass `--all`. By default, this command only prints the store paths. You can get additional information by passing flags such as `--closure-size`, diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index fc3823406..51c8a3319 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -234,9 +234,9 @@ static int main_nix_prefetch_url(int argc, char * * argv) if (!printPath) printInfo("path is '%s'", store->printStorePath(storePath)); - std::cout << printHash16or32(hash) << std::endl; + logger->cout(printHash16or32(hash)); if (printPath) - std::cout << store->printStorePath(storePath) << std::endl; + logger->cout(store->printStorePath(storePath)); return 0; } diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md index 2aad491de..a8ce9d36a 100644 --- a/src/nix/print-dev-env.md +++ b/src/nix/print-dev-env.md @@ -40,7 +40,7 @@ R""( This command prints a shell script that can be sourced by `bash` and that sets the variables and shell functions defined by the build -process of *installable*. This allows you to get a similar build +process of [*installable*](./nix.md#installables). This allows you to get a similar build environment in your current shell rather than in a subshell (as with `nix develop`). diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md index aed414963..4c0f82c09 100644 --- a/src/nix/profile-install.md +++ b/src/nix/profile-install.md @@ -29,6 +29,6 @@ R""( # Description -This command adds *installables* to a Nix profile. +This command adds [*installables*](./nix.md#installables) to a Nix profile. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 32364e720..88d4a3ce5 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -227,12 +228,12 @@ struct ProfileManifest while (i != prevElems.end() || j != curElems.end()) { if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { - std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + logger->cout("%s%s: ∅ -> %s", indent, j->describe(), j->versions()); changes = true; ++j; } else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { - std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + logger->cout("%s%s: %s -> ∅", indent, i->describe(), i->versions()); changes = true; ++i; } @@ -240,7 +241,7 @@ struct ProfileManifest auto v1 = i->versions(); auto v2 = j->versions(); if (v1 != v2) { - std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + logger->cout("%s%s: %s -> %s", indent, i->describe(), v1, v2); changes = true; } ++i; @@ -249,17 +250,23 @@ struct ProfileManifest } if (!changes) - std::cout << fmt("%sNo changes.\n", indent); + logger->cout("%sNo changes.", indent); } }; -static std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> +static std::map<Installable *, std::pair<BuiltPaths, ref<ExtraPathInfo>>> builtPathsPerInstallable( - const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths) + const std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> & builtPaths) { - std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> res; + std::map<Installable *, std::pair<BuiltPaths, ref<ExtraPathInfo>>> res; for (auto & [installable, builtPath] : builtPaths) { - auto & r = res[installable.get()]; + auto & r = res.insert({ + &*installable, + { + {}, + make_ref<ExtraPathInfo>(), + } + }).first->second; /* Note that there could be conflicting info (e.g. meta.priority fields) if the installable returned multiple derivations. So pick one arbitrarily. FIXME: @@ -295,7 +302,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile ; } - void run(ref<Store> store) override + void run(ref<Store> store, Installables && installables) override { ProfileManifest manifest(*getEvalState(), *profile); @@ -306,14 +313,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; - auto & [res, info] = builtPaths[installable.get()]; + auto iter = builtPaths.find(&*installable); + if (iter == builtPaths.end()) continue; + auto & [res, info] = iter->second; - if (info.originalRef && info.resolvedRef && info.attrPath && info.extendedOutputsSpec) { + if (auto * info2 = dynamic_cast<ExtraPathInfoFlake *>(&*info)) { element.source = ProfileElementSource { - .originalRef = *info.originalRef, - .resolvedRef = *info.resolvedRef, - .attrPath = *info.attrPath, - .outputs = *info.extendedOutputsSpec, + .originalRef = info2->flake.originalRef, + .resolvedRef = info2->flake.resolvedRef, + .attrPath = info2->value.attrPath, + .outputs = info2->value.extendedOutputsSpec, }; } @@ -322,14 +331,75 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.priority = priority ? *priority - : info.priority.value_or(defaultPriority); + : ({ + auto * info2 = dynamic_cast<ExtraPathInfoValue *>(&*info); + info2 + ? info2->value.priority.value_or(defaultPriority) + : defaultPriority; + }); element.updateStorePaths(getEvalStore(), store, res); manifest.elements.push_back(std::move(element)); } - updateProfile(manifest.build(store)); + try { + updateProfile(manifest.build(store)); + } catch (BuildEnvFileConflictError & conflictError) { + // FIXME use C++20 std::ranges once macOS has it + // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 + auto findRefByFilePath = [&]<typename Iterator>(Iterator begin, Iterator end) { + for (auto it = begin; it != end; it++) { + auto profileElement = *it; + for (auto & storePath : profileElement.storePaths) { + if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { + return std::pair(conflictError.fileA, profileElement.source->originalRef); + } + if (conflictError.fileB.starts_with(store->printStorePath(storePath))) { + return std::pair(conflictError.fileB, profileElement.source->originalRef); + } + } + } + throw conflictError; + }; + // There are 2 conflicting files. We need to find out which one is from the already installed package and + // which one is the package that is the new package that is being installed. + // The first matching package is the one that was already installed (original). + auto [originalConflictingFilePath, originalConflictingRef] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); + // The last matching package is the one that was going to be installed (new). + auto [newConflictingFilePath, newConflictingRef] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); + + throw Error( + "An existing package already provides the following file:\n" + "\n" + " %1%\n" + "\n" + "This is the conflicting file from the new package:\n" + "\n" + " %2%\n" + "\n" + "To remove the existing package:\n" + "\n" + " nix profile remove %3%\n" + "\n" + "The new package can also be installed next to the existing one by assigning a different priority.\n" + "The conflicting packages have a priority of %5%.\n" + "To prioritise the new package:\n" + "\n" + " nix profile install %4% --priority %6%\n" + "\n" + "To prioritise the existing package:\n" + "\n" + " nix profile install %4% --priority %7%\n", + originalConflictingFilePath, + newConflictingFilePath, + originalConflictingRef.to_string(), + newConflictingRef.to_string(), + conflictError.priority, + conflictError.priority - 1, + conflictError.priority + 1 + ); + } } }; @@ -456,7 +526,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); - std::vector<std::shared_ptr<Installable>> installables; + Installables installables; std::vector<size_t> indices; auto upgradedCount = 0; @@ -472,7 +542,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Activity act(*logger, lvlChatty, actUnknown, fmt("checking '%s' for updates", element.source->attrPath)); - auto installable = std::make_shared<InstallableFlake>( + auto installable = make_ref<InstallableFlake>( this, getEvalState(), FlakeRef(element.source->originalRef), @@ -484,19 +554,20 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto derivedPaths = installable->toDerivedPaths(); if (derivedPaths.empty()) continue; - auto & info = derivedPaths[0].info; - - assert(info.resolvedRef && info.attrPath); + auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; - if (element.source->resolvedRef == info.resolvedRef) continue; + if (element.source->resolvedRef == info.flake.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, *info.resolvedRef); + element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); element.source = ProfileElementSource { .originalRef = installable->flakeRef, - .resolvedRef = *info.resolvedRef, - .attrPath = *info.attrPath, + .resolvedRef = info.flake.resolvedRef, + .attrPath = info.value.attrPath, .outputs = installable->extendedOutputsSpec, }; @@ -525,7 +596,10 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first); + element.updateStorePaths( + getEvalStore(), + store, + builtPaths.find(&*installable)->second.first); } updateProfile(manifest.build(store)); @@ -583,9 +657,9 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile for (auto & gen : gens) { if (prevGen) { - if (!first) std::cout << "\n"; + if (!first) logger->cout(""); first = false; - std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); + logger->cout("Version %d -> %d:", prevGen->number, gen.number); printClosureDiff(store, store->followLinksToStorePath(prevGen->path), store->followLinksToStorePath(gen.path), @@ -621,10 +695,10 @@ struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile for (auto & gen : gens) { ProfileManifest manifest(*getEvalState(), gen.path); - if (!first) std::cout << "\n"; + if (!first) logger->cout(""); first = false; - std::cout << fmt("Version %s%d" ANSI_NORMAL " (%s)%s:\n", + logger->cout("Version %s%d" ANSI_NORMAL " (%s)%s:", gen.number == curGen ? ANSI_GREEN : ANSI_BOLD, gen.number, std::put_time(std::gmtime(&gen.creationTime), "%Y-%m-%d"), @@ -741,7 +815,6 @@ struct CmdProfile : NixMultiCommand { if (!command) throw UsageError("'nix profile' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff --git a/src/nix/profile.md b/src/nix/profile.md index 273e02280..bf61ef4b9 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -12,7 +12,7 @@ them to be rolled back easily. The default profile used by `nix profile` is `$HOME/.nix-profile`, which, if it does not exist, is created as a symlink to `/nix/var/nix/profiles/default` if Nix is invoked by the -`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. +`root` user, or `${XDG_STATE_HOME-$HOME/.local/state}/nix/profiles/profile` otherwise. You can specify another profile location using `--profile` *path*. @@ -24,11 +24,11 @@ the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix store. For example: ```console -$ ls -l /nix/var/nix/profiles/per-user/alice/profile* -lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link -lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile -lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile -lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +$ ls -l ~alice/.local/state/nix/profiles/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /home/alice/.local/state/nix/profiles/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /home/alice/.local/state/nix/profiles/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /home/alice/.local/state/nix/profiles/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile ``` Each of these symlinks is a root for the Nix garbage collector. @@ -38,20 +38,20 @@ profile is a tree of symlinks to the files of the installed packages, e.g. ```console -$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ -/nix/var/nix/profiles/per-user/eelco/profile-7-link/: +$ ll -R ~eelco/.local/state/nix/profiles/profile-7-link/ +/home/eelco/.local/state/nix/profiles/profile-7-link/: total 20 dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin -r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json dr-xr-xr-x 4 root root 4096 Jan 1 1970 share -/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: +/home/eelco/.local/state/nix/profiles/profile-7-link/bin: total 20 lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us -/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: +/home/eelco/.local/state/nix/profiles/profile-7-link/share/applications: total 12 lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index c9a7157cd..e19e93219 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -21,7 +21,6 @@ struct CmdRealisation : virtual NixMultiCommand { if (!command) throw UsageError("'nix realisation' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; @@ -46,7 +45,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON void run(ref<Store> store, BuiltPaths && paths) override { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); RealisedPath::Set realisations; for (auto & builtPath : paths) { @@ -65,18 +64,16 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON res.push_back(currentPath); } - std::cout << res.dump(); + logger->cout("%s", res); } else { for (auto & path : realisations) { if (auto realisation = std::get_if<Realisation>(&path.raw)) { - std::cout << - realisation->id.to_string() << " " << - store->printStorePath(realisation->outPath); + logger->cout("%s %s", + realisation->id.to_string(), + store->printStorePath(realisation->outPath)); } else - std::cout << store->printStorePath(path.path()); - - std::cout << std::endl; + logger->cout("%s", store->printStorePath(path.path())); } } } diff --git a/src/nix/registry.cc b/src/nix/registry.cc index b5bdfba95..cb94bbd31 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -224,10 +224,9 @@ struct CmdRegistry : virtual NixMultiCommand void run() override { - settings.requireExperimentalFeature(Xp::Flakes); + experimentalFeatureSettings.require(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 000000000..bb14f3f99 --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,102 @@ +#include "eval.hh" +#include "globals.hh" +#include "command.hh" +#include "installable-value.hh" +#include "repl.hh" + +namespace nix { + +struct CmdRepl : RawInstallablesCommand +{ + CmdRepl() { + evalSettings.pureEval = false; + } + + /** + * This command is stable before the others + */ + std::optional<ExperimentalFeature> experimentalFeature() override + { + return std::nullopt; + } + + std::vector<std::string> files; + + Strings getDefaultFlakeAttrPaths() override + { + return {""}; + } + + bool forceImpureByDefault() override + { + return true; + } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + std::string doc() override + { + return + #include "repl.md" + ; + } + + void applyDefaultInstallables(std::vector<std::string> & rawInstallables) override + { + if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + warn("future versions of Nix will require using `--file` to load a file"); + if (rawInstallables.size() > 1) + warn("more than one input file is not currently supported"); + auto filePath = rawInstallables[0].data(); + file = std::optional(filePath); + rawInstallables.front() = rawInstallables.back(); + rawInstallables.pop_back(); + } + if (rawInstallables.empty() && (file.has_value() || expr.has_value())) { + rawInstallables.push_back("."); + } + } + + void run(ref<Store> store, std::vector<std::string> && rawInstallables) override + { + auto state = getEvalState(); + auto getValues = [&]()->AbstractNixRepl::AnnotatedValues{ + auto installables = parseInstallables(store, rawInstallables); + AbstractNixRepl::AnnotatedValues values; + for (auto & installable_: installables){ + auto & installable = InstallableValue::require(*installable_); + auto what = installable.what(); + if (file){ + auto [val, pos] = installable.toValue(*state); + auto what = installable.what(); + state->forceValue(*val, pos); + auto autoArgs = getAutoArgs(*state); + auto valPost = state->allocValue(); + state->autoCallFunction(*autoArgs, *val, *valPost); + state->forceValue(*valPost, pos); + values.push_back( {valPost, what }); + } else { + auto [val, pos] = installable.toValue(*state); + values.push_back( {val, what} ); + } + } + return values; + }; + auto repl = AbstractNixRepl::create( + searchPath, + openStore(), + state, + getValues + ); + repl->autoArgs = getAutoArgs(*repl->state); + repl->initEnv(); + repl->mainLoop(); + } +}; + +static auto rCmdRepl = registerCommand<CmdRepl>("repl"); + +} diff --git a/src/nix/run.cc b/src/nix/run.cc index 6fca68047..1baf299ab 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,5 +1,5 @@ #include "run.hh" -#include "command.hh" +#include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -97,7 +97,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment ; } - void run(ref<Store> store) override + void run(ref<Store> store, Installables && installables) override { auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); @@ -137,7 +137,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment static auto rCmdShell = registerCommand<CmdShell>("shell"); -struct CmdRun : InstallableCommand +struct CmdRun : InstallableValueCommand { using InstallableCommand::run; @@ -183,7 +183,7 @@ struct CmdRun : InstallableCommand return res; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { auto state = getEvalState(); diff --git a/src/nix/run.hh b/src/nix/run.hh index fed360158..97ddef19b 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/nix/run.md b/src/nix/run.md index a0f362076..250ea65aa 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -35,7 +35,7 @@ R""( # Description -`nix run` builds and runs *installable*, which must evaluate to an +`nix run` builds and runs [*installable*](./nix.md#installables), which must evaluate to an *app* or a regular Nix derivation. If *installable* evaluates to an *app* (see below), it executes the diff --git a/src/nix/search.cc b/src/nix/search.cc index 4fa1e7837..c92ed1663 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -1,4 +1,4 @@ -#include "command.hh" +#include "command-installable-value.hh" #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" @@ -22,7 +22,7 @@ std::string wrap(std::string prefix, std::string s) return concatStrings(prefix, s, ANSI_NORMAL); } -struct CmdSearch : InstallableCommand, MixJSON +struct CmdSearch : InstallableValueCommand, MixJSON { std::vector<std::string> res; std::vector<std::string> excludeRes; @@ -61,7 +61,7 @@ struct CmdSearch : InstallableCommand, MixJSON }; } - void run(ref<Store> store) override + void run(ref<Store> store, ref<InstallableValue> installable) override { settings.readOnlyMode = true; evalSettings.enableImportFromDerivation.setDefault(false); @@ -196,9 +196,8 @@ struct CmdSearch : InstallableCommand, MixJSON for (auto & cursor : installable->getCursors(*state)) visit(*cursor, cursor->getAttrPath(), true); - if (json) { - std::cout << jsonOut->dump() << std::endl; - } + if (json) + logger->cout("%s", *jsonOut); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/src/nix/search.md b/src/nix/search.md index 5a5b5ae05..4caa90654 100644 --- a/src/nix/search.md +++ b/src/nix/search.md @@ -62,10 +62,10 @@ R""( # Description -`nix search` searches *installable* (which must be evaluatable, e.g. a -flake) for packages whose name or description matches all of the +`nix search` searches [*installable*](./nix.md#installables) (which can be evaluated, that is, a +flake or Nix expression, but not a store path or store derivation path) for packages whose name or description matches all of the regular expressions *regex*. For each matching package, It prints the -full attribute name (from the root of the installable), the version +full attribute name (from the root of the [installable](./nix.md#installables)), the version and the `meta.description` field, highlighting the substrings that were matched by the regular expressions. If no regular expressions are specified, all packages are shown. diff --git a/src/nix/shell.md b/src/nix/shell.md index 9fa1031f5..13a389103 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -48,7 +48,7 @@ R""( # Description `nix shell` runs a command in an environment in which the `$PATH` variable -provides the specified *installables*. If no command is specified, it starts the +provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. )"" diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index af2e676a4..4a406ae08 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -39,7 +39,7 @@ struct CmdShowDerivation : InstallablesCommand Category category() override { return catUtility; } - void run(ref<Store> store) override + void run(ref<Store> store, Installables && installables) override { auto drvPaths = Installable::toDerivations(store, installables, true); @@ -54,58 +54,10 @@ struct CmdShowDerivation : InstallablesCommand for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - json& drvObj = jsonRoot[store->printStorePath(drvPath)]; - - auto drv = store->readDerivation(drvPath); - - { - json& outputsObj = drvObj["outputs"]; - outputsObj = json::object(); - for (auto & [_outputName, output] : drv.outputs) { - auto & outputName = _outputName; // work around clang bug - auto& outputObj = outputsObj[outputName]; - outputObj = json::object(); - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doi) { - outputObj["path"] = store->printStorePath(doi.path); - }, - [&](const DerivationOutput::CAFixed & dof) { - outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName)); - outputObj["hashAlgo"] = dof.hash.printMethodAlgo(); - outputObj["hash"] = dof.hash.hash.to_string(Base16, false); - }, - [&](const DerivationOutput::CAFloating & dof) { - outputObj["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); - }, - [&](const DerivationOutput::Deferred &) {}, - [&](const DerivationOutput::Impure & doi) { - outputObj["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); - outputObj["impure"] = true; - }, - }, output.raw()); - } - } - - { - auto& inputsList = drvObj["inputSrcs"]; - inputsList = json::array(); - for (auto & input : drv.inputSrcs) - inputsList.emplace_back(store->printStorePath(input)); - } - - { - auto& inputDrvsObj = drvObj["inputDrvs"]; - inputDrvsObj = json::object(); - for (auto & input : drv.inputDrvs) - inputDrvsObj[store->printStorePath(input.first)] = input.second; - } - - drvObj["system"] = drv.platform; - drvObj["builder"] = drv.builder; - drvObj["args"] = drv.args; - drvObj["env"] = drv.env; + jsonRoot[store->printStorePath(drvPath)] = + store->readDerivation(drvPath).toJSON(*store); } - std::cout << jsonRoot.dump(2) << std::endl; + logger->cout(jsonRoot.dump(2)); } }; diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md index 2cd93aa62..1d37c6f5a 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/show-derivation.md @@ -39,7 +39,7 @@ R""( # Description This command prints on standard output a JSON representation of the -[store derivation]s to which *installables* evaluate. Store derivations +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations are used internally by Nix. They are store paths with extension `.drv` that represent the build-time dependency graph to which a Nix expression evaluates. diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 3d659d6d2..45cd2e1a6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -45,7 +45,7 @@ struct CmdCopySigs : StorePathsCommand //logger->setExpected(doneLabel, storePaths.size()); auto doPath = [&](const Path & storePathS) { - //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); + //Activity act(*logger, lvlInfo, "getting signatures for '%s'", storePath); checkInterrupt(); @@ -173,7 +173,7 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); - std::cout << SecretKey::generate(*keyName).to_string(); + writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -194,7 +194,7 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); - std::cout << secretKey.toPublicKey().to_string(); + writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; @@ -219,7 +219,6 @@ struct CmdKey : NixMultiCommand { if (!command) throw UsageError("'nix key' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index d5fab5f2f..a6e8aeff7 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -24,9 +24,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand ; } - Category category() override { return catUtility; } - - void run(ref<Store> srcStore) override + void run(ref<Store> srcStore, Installables && installables) override { auto & srcLogStore = require<LogStore>(*srcStore); diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index ca43f1530..6719227df 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -32,7 +32,7 @@ struct CmdStoreDelete : StorePathsCommand ; } - void run(ref<Store> store, std::vector<StorePath> && storePaths) override + void run(ref<Store> store, StorePaths && storePaths) override { auto & gcStore = require<GcStore>(*store); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md index db535f87c..431bc5f5e 100644 --- a/src/nix/store-delete.md +++ b/src/nix/store-delete.md @@ -10,7 +10,7 @@ R""( # Description -This command deletes the store paths specified by *installables*. , +This command deletes the store paths specified by [*installables*](./nix.md#installables), but only if it is safe to do so; that is, when the path is not reachable from a root of the garbage collector. This means that you can only delete paths that would also be deleted by `nix store diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md index 4ef563526..56e2174b6 100644 --- a/src/nix/store-dump-path.md +++ b/src/nix/store-dump-path.md @@ -18,6 +18,6 @@ R""( # Description This command generates a NAR file containing the serialisation of the -store path *installable*. The NAR is written to standard output. +store path [*installable*](./nix.md#installables). The NAR is written to standard output. )"" diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc index 8fcb3639a..895e39685 100644 --- a/src/nix/store-repair.cc +++ b/src/nix/store-repair.cc @@ -17,7 +17,7 @@ struct CmdStoreRepair : StorePathsCommand ; } - void run(ref<Store> store, std::vector<StorePath> && storePaths) override + void run(ref<Store> store, StorePaths && storePaths) override { for (auto & path : storePaths) store->repairPath(path); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md index 92d2205a9..180c577ac 100644 --- a/src/nix/store-repair.md +++ b/src/nix/store-repair.md @@ -17,7 +17,7 @@ R""( # Description This command attempts to "repair" the store paths specified by -*installables* by redownloading them using the available +[*installables*](./nix.md#installables) by redownloading them using the available substituters. If no substitutes are available, then repair is not possible. diff --git a/src/nix/store.cc b/src/nix/store.cc index 44e53c7c7..2879e03b3 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -18,7 +18,6 @@ struct CmdStore : virtual NixMultiCommand { if (!command) throw UsageError("'nix store' requires a sub-command."); - command->second->prepare(); command->second->run(); } }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17796d6b8..2295d86d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -32,6 +32,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }); } + /** + * This command is stable before the others + */ + std::optional<ExperimentalFeature> experimentalFeature() override + { + return std::nullopt; + } + std::string description() override { return "upgrade Nix to the stable version declared in Nixpkgs"; diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 084c80ba2..08757aebd 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -11,7 +11,7 @@ R""( * Upgrade Nix in a specific profile: ```console - # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + # nix upgrade-nix -p ~alice/.local/state/nix/profiles/profile ``` # Description diff --git a/src/nix/verify.md b/src/nix/verify.md index 1c43792e7..cc1122c02 100644 --- a/src/nix/verify.md +++ b/src/nix/verify.md @@ -24,7 +24,7 @@ R""( # Description -This command verifies the integrity of the store paths *installables*, +This command verifies the integrity of the store paths [*installables*](./nix.md#installables), or, if `--all` is given, the entire Nix store. For each path, it checks that diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index c6023eb03..4ea268d24 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -157,13 +157,9 @@ int main(int argc, char ** argv) uname(&_uname); - auto cacheParentDir = (format("%1%/dependency-maps") % settings.nixStateDir).str(); + auto cacheParentDir = fmt("%1%/dependency-maps", settings.nixStateDir); - cacheDir = (format("%1%/%2%-%3%-%4%") - % cacheParentDir - % _uname.machine - % _uname.sysname - % _uname.release).str(); + cacheDir = fmt("%1%/%2%-%3%-%4%", cacheParentDir, _uname.machine, _uname.sysname, _uname.release); mkdir(cacheParentDir.c_str(), 0755); mkdir(cacheDir.c_str(), 0755); |