diff options
author | John Ericson <John.Ericson@Obsidian.Systems> | 2023-02-28 12:46:00 -0500 |
---|---|---|
committer | John Ericson <John.Ericson@Obsidian.Systems> | 2023-02-28 12:46:00 -0500 |
commit | 5abd643c6d10f2cfa6e26652a9688a0263310094 (patch) | |
tree | 2fdb8bf147cb93430ba3ba79a473568e2584e497 /src | |
parent | e68e8e3cee53ce7debd7c54b0d122d94d1b102a2 (diff) | |
parent | d381248ec0847cacd918480e83a99287f814456a (diff) |
Merge branch 'path-info' into ca-drv-exotic
Diffstat (limited to 'src')
117 files changed, 2376 insertions, 1057 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 6b81ecc49..174435e7c 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -72,6 +72,7 @@ static int main_build_remote(int argc, char * * argv) settings.set(name, value); } + auto maxBuildJobs = settings.maxBuildJobs; settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work initPlugins(); @@ -112,10 +113,14 @@ static int main_build_remote(int argc, char * * argv) drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling + /* It would be possible to build locally after some builds clear out, + so don't show the warning now: */ + bool couldBuildLocally = maxBuildJobs > 0 && ( neededSystem == settings.thisSystem || settings.extraPlatforms.get().count(neededSystem) > 0) && allSupportedLocally(*store, requiredFeatures); + /* It's possible to build this locally right now: */ + bool canBuildLocally = amWilling && couldBuildLocally; /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -214,7 +219,7 @@ static int main_build_remote(int argc, char * * argv) % concatStringsSep<StringSet>(", ", m.supportedFeatures) % concatStringsSep<StringSet>(", ", m.mandatoryFeatures); - printMsg(canBuildLocally ? lvlChatty : lvlWarn, error); + printMsg(couldBuildLocally ? lvlChatty : lvlWarn, error); std::cerr << "# decline\n"; } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 0740ea960..ab51c229d 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -4,6 +4,7 @@ #include "derivations.hh" #include "nixexpr.hh" #include "profiles.hh" +#include "repl.hh" #include <nlohmann/json.hpp> @@ -121,12 +122,22 @@ ref<EvalState> EvalCommand::getEvalState() ; if (startReplOnEvalErrors) { - evalState->debugRepl = &runRepl; + evalState->debugRepl = &AbstractNixRepl::runSimple; }; } return ref<EvalState>(evalState); } +MixOperateOnOptions::MixOperateOnOptions() +{ + addFlag({ + .longName = "derivation", + .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .category = installablesCategory, + .handler = {&operateOn, OperateOn::Derivation}, + }); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { @@ -208,20 +219,6 @@ void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePath run(store, *storePaths.begin()); } -Strings editorFor(const Path & file, uint32_t line) -{ - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString<Strings>(editor); - if (line > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos || - editor.find("kak") != std::string::npos)) - args.push_back(fmt("+%d", line)); - args.push_back(file); - return args; -} - MixProfile::MixProfile() { addFlag({ diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 3b4b40981..b6d554aab 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -94,12 +94,8 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional<Path> file; std::optional<std::string> expr; - bool readOnlyMode = false; - // FIXME: move this; not all commands (e.g. 'nix run') use it. - OperateOn operateOn = OperateOn::Output; - - SourceExprCommand(bool supportReadOnlyMode = false); + SourceExprCommand(); std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); @@ -114,6 +110,11 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions void completeInstallable(std::string_view prefix); }; +struct MixReadOnlyOption : virtual Args +{ + MixReadOnlyOption(); +}; + /* A command that operates on a list of "installables", which can be store paths, attribute paths, Nix expressions, etc. */ struct InstallablesCommand : virtual Args, SourceExprCommand @@ -139,7 +140,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr<Installable> installable; - InstallableCommand(bool supportReadOnlyMode = false); + InstallableCommand(); void prepare() override; @@ -153,8 +154,15 @@ private: std::string _installable{"."}; }; +struct MixOperateOnOptions : virtual Args +{ + OperateOn operateOn = OperateOn::Output; + + MixOperateOnOptions(); +}; + /* A command that operates on zero or more store paths. */ -struct BuiltPathsCommand : public InstallablesCommand +struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -227,10 +235,6 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name) return RegisterCommand(std::move(name), [](){ return make_ref<T>(); }); } -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ -Strings editorFor(const Path & file, uint32_t line); - struct MixProfile : virtual StoreCommand { std::optional<Path> profile; @@ -280,8 +284,4 @@ void printClosureDiff( const StorePath & afterPath, std::string_view indent); - -void runRepl( - ref<EvalState> evalState, - const ValMap & extraEnv); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc new file mode 100644 index 000000000..f674f32bd --- /dev/null +++ b/src/libcmd/editor-for.cc @@ -0,0 +1,20 @@ +#include "util.hh" +#include "editor-for.hh" + +namespace nix { + +Strings editorFor(const Path & file, uint32_t line) +{ + auto editor = getEnv("EDITOR").value_or("cat"); + auto args = tokenizeString<Strings>(editor); + if (line > 0 && ( + editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos || + editor.find("kak") != std::string::npos)) + args.push_back(fmt("+%d", line)); + args.push_back(file); + return args; +} + +} diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh new file mode 100644 index 000000000..8fbd08792 --- /dev/null +++ b/src/libcmd/editor-for.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +/* Helper function to generate args that invoke $EDITOR on + filename:lineno. */ +Strings editorFor(const Path & file, uint32_t line); + +} diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc new file mode 100644 index 000000000..d9377f0d6 --- /dev/null +++ b/src/libcmd/installable-attr-path.cc @@ -0,0 +1,109 @@ +#include "globals.hh" +#include "installable-attr-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +InstallableAttrPath::InstallableAttrPath( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec) + : InstallableValue(state) + , cmd(cmd) + , v(allocRootValue(v)) + , attrPath(attrPath) + , extendedOutputsSpec(std::move(extendedOutputsSpec)) +{ } + +std::pair<Value *, PosIdx> InstallableAttrPath::toValue(EvalState & state) +{ + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); + state.forceValue(*vRes, pos); + return {vRes, pos}; +} + +DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() +{ + auto v = toValue(*state).first; + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); + + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map<StorePath, OutputsSpec> byDrvPath; + + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); + + auto newOutputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + for (auto & output : drvInfo.queryOutputs(false, true)) + outputsToInstall.insert(output.first); + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()); + + auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); + + if (!didInsert) + iter->second = iter->second.union_(newOutputs); + } + + DerivedPathsWithInfo res; + for (auto & [drvPath, outputs] : byDrvPath) + res.push_back({ + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = outputs, + }, + }); + + return res; +} + +InstallableAttrPath InstallableAttrPath::parse( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + return { + state, cmd, v, + prefix == "." ? "" : std::string { prefix }, + extendedOutputsSpec + }; +} + +} diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh new file mode 100644 index 000000000..c06132ec8 --- /dev/null +++ b/src/libcmd/installable-attr-path.hh @@ -0,0 +1,56 @@ +#include "globals.hh" +#include "installable-value.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +class InstallableAttrPath : public InstallableValue +{ + SourceExprCommand & cmd; + RootValue v; + std::string attrPath; + ExtendedOutputsSpec extendedOutputsSpec; + + InstallableAttrPath( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + const std::string & attrPath, + ExtendedOutputsSpec extendedOutputsSpec); + + std::string what() const override { return attrPath; }; + + std::pair<Value *, PosIdx> toValue(EvalState & state) override; + + DerivedPathsWithInfo toDerivedPaths() override; + +public: + + static InstallableAttrPath parse( + ref<EvalState> state, + SourceExprCommand & cmd, + Value * v, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc new file mode 100644 index 000000000..a9921b901 --- /dev/null +++ b/src/libcmd/installable-derived-path.cc @@ -0,0 +1,70 @@ +#include "installable-derived-path.hh" +#include "derivations.hh" + +namespace nix { + +std::string InstallableDerivedPath::what() const +{ + return derivedPath.to_string(*store); +} + +DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() +{ + return {{.path = derivedPath, .info = {} }}; +} + +std::optional<StorePath> InstallableDerivedPath::getStorePath() +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + [&](const DerivedPath::Opaque & bo) { + return bo.path; + }, + }, derivedPath.raw()); +} + +InstallableDerivedPath InstallableDerivedPath::parse( + ref<Store> store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec) +{ + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more liberally. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + // First, we accept a symlink chain or an actual store path. + auto storePath = store->followLinksToStorePath(prefix); + // Second, we see if the store path ends in `.drv` to decide what sort + // of derived path they want. + // + // This handling predates the `^` syntax. The `^*` in + // `/nix/store/hash-foo.drv^*` unambiguously means "do the + // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could + // also unambiguously mean "do the DerivedPath::Opaque` case". + // + // Issue #7261 tracks reconsidering this `.drv` dispatching. + return storePath.isDerivation() + ? (DerivedPath) DerivedPath::Built { + .drvPath = std::move(storePath), + .outputs = OutputsSpec::All {}, + } + : (DerivedPath) DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + return DerivedPath::Built { + .drvPath = store->parseStorePath(prefix), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw()); + return InstallableDerivedPath { + store, + std::move(derivedPath), + }; +} + +} diff --git a/src/libcmd/installable-derived-path.hh b/src/libcmd/installable-derived-path.hh new file mode 100644 index 000000000..042878b91 --- /dev/null +++ b/src/libcmd/installable-derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "installables.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref<Store> store; + DerivedPath derivedPath; + + InstallableDerivedPath(ref<Store> store, DerivedPath && derivedPath) + : store(store), derivedPath(std::move(derivedPath)) + { } + + std::string what() const override; + + DerivedPathsWithInfo toDerivedPaths() override; + + std::optional<StorePath> getStorePath() override; + + static InstallableDerivedPath parse( + ref<Store> store, + std::string_view prefix, + ExtendedOutputsSpec extendedOutputsSpec); +}; + +} diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc new file mode 100644 index 000000000..60a97deaf --- /dev/null +++ b/src/libcmd/installable-flake.cc @@ -0,0 +1,236 @@ +#include "globals.hh" +#include "installable-flake.hh" +#include "installable-derived-path.hh" +#include "outputs-spec.hh" +#include "util.hh" +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" +#include "flake/flake.hh" +#include "eval-cache.hh" +#include "url.hh" +#include "registry.hh" +#include "build-result.hh" + +#include <regex> +#include <queue> + +#include <nlohmann/json.hpp> + +namespace nix { + +std::vector<std::string> InstallableFlake::getActualAttrPaths() +{ + std::vector<std::string> res; + + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); + + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); + + return aOutputs->value; +} + +static std::string showAttrPaths(const std::vector<std::string> & paths) +{ + std::string s; + for (const auto & [n, i] : enumerate(paths)) { + if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; + s += '\''; s += i; s += '\''; + } + return s; +} + +InstallableFlake::InstallableFlake( + SourceExprCommand * cmd, + ref<EvalState> state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags) + : InstallableValue(state), + flakeRef(flakeRef), + attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), + prefixes(fragment == "" ? Strings{} : prefixes), + extendedOutputsSpec(std::move(extendedOutputsSpec)), + lockFlags(lockFlags) +{ + if (cmd && cmd->getAutoArgs(*state)->size()) + throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); +} + +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() +{ + Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); + + auto attr = getCursor(*state); + + auto attrPath = attr->getAttrPathStr(); + + if (!attr->isDerivation()) { + + // FIXME: use eval cache? + auto v = attr->forceValue(); + + if (v.type() == nPath) { + PathSet context; + auto storePath = state->copyPathToStore(context, Path(v.path)); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + } + }}; + } + + else if (v.type() == nString) { + PathSet context; + auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); + auto storePath = state->store->maybeParseStorePath(s); + if (storePath && context.count(std::string(s))) { + return {{ + .path = DerivedPath::Opaque { + .path = std::move(*storePath), + } + }}; + } else + throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); + } + + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); + } + + auto drvPath = attr->forceDerivation(); + + std::optional<NixInt> priority; + + if (attr->maybeGetAttr(state->sOutputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } + + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (aOutputSpecified->getBool()) { + if (auto aOutputName = attr->maybeGetAttr("outputName")) + outputsToInstall = { aOutputName->getString() }; + } + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) + for (auto & s : aOutputsToInstall->getListOfStrings()) + outputsToInstall.insert(s); + } + + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); + + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()), + }, + .info = { + .priority = priority, + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + } + }}; +} + +std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) +{ + return {&getCursor(state)->forceValue(), noPos}; +} + +std::vector<ref<eval_cache::AttrCursor>> +InstallableFlake::getCursors(EvalState & state) +{ + auto evalCache = openEvalCache(state, + std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags))); + + auto root = evalCache->getRoot(); + + std::vector<ref<eval_cache::AttrCursor>> res; + + Suggestions suggestions; + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); + if (attr) { + res.push_back(ref(*attr)); + } else { + suggestions += attr.getSuggestions(); + } + } + + if (res.size() == 0) + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); + + return res; +} + +std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const +{ + if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; + _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + } + return _lockedFlake; +} + +FlakeRef InstallableFlake::nixpkgsFlakeRef() const +{ + auto lockedFlake = getLockedFlake(); + + if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { + if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { + debug("using nixpkgs flake '%s'", lockedNode->lockedRef); + return std::move(lockedNode->lockedRef); + } + } + + return Installable::nixpkgsFlakeRef(); +} + +} diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh new file mode 100644 index 000000000..c75765086 --- /dev/null +++ b/src/libcmd/installable-flake.hh @@ -0,0 +1,50 @@ +#pragma once + +#include "installable-value.hh" + +namespace nix { + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + ExtendedOutputsSpec extendedOutputsSpec; + const flake::LockFlags & lockFlags; + mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; + + InstallableFlake( + SourceExprCommand * cmd, + ref<EvalState> state, + FlakeRef && flakeRef, + std::string_view fragment, + ExtendedOutputsSpec extendedOutputsSpec, + Strings attrPaths, + Strings prefixes, + const flake::LockFlags & lockFlags); + + std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector<std::string> getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + DerivedPathsWithInfo toDerivedPaths() override; + + std::pair<Value *, PosIdx> toValue(EvalState & state) override; + + /* Get a cursor to every attrpath in getActualAttrPaths() + that exists. However if none exists, throw an exception. */ + std::vector<ref<eval_cache::AttrCursor>> + getCursors(EvalState & state) override; + + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; + + FlakeRef nixpkgsFlakeRef() const override; +}; + +ref<eval_cache::EvalCache> openEvalCache( + EvalState & state, + std::shared_ptr<flake::LockedFlake> lockedFlake); + +} diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh new file mode 100644 index 000000000..c6cdc4797 --- /dev/null +++ b/src/libcmd/installable-value.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "installables.hh" + +namespace nix { + +struct InstallableValue : Installable +{ + ref<EvalState> state; + + InstallableValue(ref<EvalState> state) : state(state) {} +}; + +} diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5090ea6d2..00c6f9516 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,5 +1,8 @@ #include "globals.hh" #include "installables.hh" +#include "installable-derived-path.hh" +#include "installable-attr-path.hh" +#include "installable-flake.hh" #include "outputs-spec.hh" #include "util.hh" #include "command.hh" @@ -144,7 +147,7 @@ void MixFlakeOptions::completionHook() completeFlakeInput(*prefix); } -SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) +SourceExprCommand::SourceExprCommand() { addFlag({ .longName = "file", @@ -166,24 +169,18 @@ SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) .labels = {"expr"}, .handler = {&expr} }); +} +MixReadOnlyOption::MixReadOnlyOption() +{ addFlag({ - .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", - .category = installablesCategory, - .handler = {&operateOn, OperateOn::Derivation}, + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&settings.readOnlyMode, true}, }); - - if (supportReadOnlyMode) { - addFlag({ - .longName = "read-only", - .description = - "Do not instantiate each evaluated derivation. " - "This improves performance, but can cause errors when accessing " - "store paths of derivations during evaluation.", - .handler = {&readOnlyMode, true}, - }); - } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -379,10 +376,9 @@ Installable::getCursors(EvalState & state) ref<eval_cache::AttrCursor> Installable::getCursor(EvalState & state) { - auto cursors = getCursors(state); - if (cursors.empty()) - throw Error("cannot find flake attribute '%s'", what()); - return cursors[0]; + /* Although getCursors should return at least one element, in case it doesn't, + bound check to avoid an undefined behavior for vector[0] */ + return getCursors(state).at(0); } static StorePath getDeriver( @@ -397,143 +393,6 @@ static StorePath getDeriver( return *derivers.begin(); } -struct InstallableStorePath : Installable -{ - ref<Store> store; - DerivedPath req; - - InstallableStorePath(ref<Store> store, DerivedPath && req) - : store(store), req(std::move(req)) - { } - - std::string what() const override - { - return req.to_string(*store); - } - - DerivedPathsWithInfo toDerivedPaths() override - { - return {{.path = req, .info = {} }}; - } - - std::optional<StorePath> getStorePath() override - { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, req.raw()); - } -}; - -struct InstallableAttrPath : InstallableValue -{ - SourceExprCommand & cmd; - RootValue v; - std::string attrPath; - ExtendedOutputsSpec extendedOutputsSpec; - - InstallableAttrPath( - ref<EvalState> state, - SourceExprCommand & cmd, - Value * v, - const std::string & attrPath, - ExtendedOutputsSpec extendedOutputsSpec) - : InstallableValue(state) - , cmd(cmd) - , v(allocRootValue(v)) - , attrPath(attrPath) - , extendedOutputsSpec(std::move(extendedOutputsSpec)) - { } - - std::string what() const override { return attrPath; } - - std::pair<Value *, PosIdx> toValue(EvalState & state) override - { - auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v); - state.forceValue(*vRes, pos); - return {vRes, pos}; - } - - DerivedPathsWithInfo toDerivedPaths() override - { - auto v = toValue(*state).first; - - Bindings & autoArgs = *cmd.getAutoArgs(*state); - - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); - - // Backward compatibility hack: group results by drvPath. This - // helps keep .all output together. - std::map<StorePath, OutputsSpec> byDrvPath; - - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); - - auto newOutputs = std::visit(overloaded { - [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set<std::string> outputsToInstall; - for (auto & output : drvInfo.queryOutputs(false, true)) - outputsToInstall.insert(output.first); - return OutputsSpec::Names { std::move(outputsToInstall) }; - }, - [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { - return e; - }, - }, extendedOutputsSpec.raw()); - - auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); - - if (!didInsert) - iter->second = iter->second.union_(newOutputs); - } - - DerivedPathsWithInfo res; - for (auto & [drvPath, outputs] : byDrvPath) - res.push_back({ - .path = DerivedPath::Built { - .drvPath = drvPath, - .outputs = outputs, - }, - }); - - return res; - } -}; - -std::vector<std::string> InstallableFlake::getActualAttrPaths() -{ - std::vector<std::string> res; - - for (auto & prefix : prefixes) - res.push_back(prefix + *attrPaths.begin()); - - for (auto & s : attrPaths) - res.push_back(s); - - return res; -} - -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, [&]() { return aOutputs->value->determinePos(noPos); }); - - return aOutputs->value; -} - ref<eval_cache::EvalCache> openEvalCache( EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake) @@ -563,214 +422,11 @@ ref<eval_cache::EvalCache> openEvalCache( }); } -static std::string showAttrPaths(const std::vector<std::string> & paths) -{ - std::string s; - for (const auto & [n, i] : enumerate(paths)) { - if (n > 0) s += n + 1 == paths.size() ? " or " : ", "; - s += '\''; s += i; s += '\''; - } - return s; -} - -InstallableFlake::InstallableFlake( - SourceExprCommand * cmd, - ref<EvalState> state, - FlakeRef && flakeRef, - std::string_view fragment, - ExtendedOutputsSpec extendedOutputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags) - : InstallableValue(state), - flakeRef(flakeRef), - attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), - prefixes(fragment == "" ? Strings{} : prefixes), - extendedOutputsSpec(std::move(extendedOutputsSpec)), - lockFlags(lockFlags) -{ - if (cmd && cmd->getAutoArgs(*state)->size()) - throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); -} - -DerivedPathsWithInfo InstallableFlake::toDerivedPaths() -{ - Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); - - auto attr = getCursor(*state); - - auto attrPath = attr->getAttrPathStr(); - - if (!attr->isDerivation()) { - - // FIXME: use eval cache? - auto v = attr->forceValue(); - - if (v.type() == nPath) { - PathSet context; - auto storePath = state->copyPathToStore(context, Path(v.path)); - return {{ - .path = DerivedPath::Opaque { - .path = std::move(storePath), - } - }}; - } - - else if (v.type() == nString) { - PathSet context; - auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); - auto storePath = state->store->maybeParseStorePath(s); - if (storePath && context.count(std::string(s))) { - return {{ - .path = DerivedPath::Opaque { - .path = std::move(*storePath), - } - }}; - } else - throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); - } - - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); - } - - auto drvPath = attr->forceDerivation(); - - std::optional<NixInt> priority; - - if (attr->maybeGetAttr(state->sOutputSpecified)) { - } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); - } - - return {{ - .path = DerivedPath::Built { - .drvPath = std::move(drvPath), - .outputs = std::visit(overloaded { - [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { - std::set<std::string> outputsToInstall; - if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { - if (aOutputSpecified->getBool()) { - if (auto aOutputName = attr->maybeGetAttr("outputName")) - outputsToInstall = { aOutputName->getString() }; - } - } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) - for (auto & s : aOutputsToInstall->getListOfStrings()) - outputsToInstall.insert(s); - } - - if (outputsToInstall.empty()) - outputsToInstall.insert("out"); - - return OutputsSpec::Names { std::move(outputsToInstall) }; - }, - [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { - return e; - }, - }, extendedOutputsSpec.raw()), - }, - .info = { - .priority = priority, - .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, - .attrPath = attrPath, - .extendedOutputsSpec = extendedOutputsSpec, - } - }}; -} - -std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) -{ - return {&getCursor(state)->forceValue(), noPos}; -} - -std::vector<ref<eval_cache::AttrCursor>> -InstallableFlake::getCursors(EvalState & state) -{ - auto evalCache = openEvalCache(state, - std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags))); - - auto root = evalCache->getRoot(); - - std::vector<ref<eval_cache::AttrCursor>> res; - - for (auto & attrPath : getActualAttrPaths()) { - auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back(ref(*attr)); - } - - return res; -} - -ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state) -{ - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(state, lockedFlake); - auto root = cache->getRoot(); - - Suggestions suggestions; - - auto attrPaths = getActualAttrPaths(); - - for (auto & attrPath : attrPaths) { - debug("trying flake output attribute '%s'", attrPath); - - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } - - return *attrOrSuggestions; - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(attrPaths)); -} - -std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const -{ - if (!_lockedFlake) { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); - } - return _lockedFlake; -} - -FlakeRef InstallableFlake::nixpkgsFlakeRef() const -{ - auto lockedFlake = getLockedFlake(); - - if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) { - if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) { - debug("using nixpkgs flake '%s'", lockedNode->lockedRef); - return std::move(lockedNode->lockedRef); - } - } - - return Installable::nixpkgsFlakeRef(); -} - std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( ref<Store> store, std::vector<std::string> ss) { std::vector<std::shared_ptr<Installable>> result; - if (readOnlyMode) { - settings.readOnlyMode = true; - } - if (file || expr) { if (file && expr) throw UsageError("'--file' and '--expr' are exclusive"); @@ -796,9 +452,8 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s); result.push_back( std::make_shared<InstallableAttrPath>( - state, *this, vFile, - prefix == "." ? "" : std::string { prefix }, - extendedOutputsSpec)); + InstallableAttrPath::parse( + state, *this, vFile, prefix, extendedOutputsSpec))); } } else { @@ -811,41 +466,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( auto prefix = std::move(prefix_); auto extendedOutputsSpec = std::move(extendedOutputsSpec_); - auto found = prefix.find('/'); - if (found != std::string::npos) { + if (prefix.find('/') != std::string::npos) { try { - auto derivedPath = std::visit(overloaded { - // If the user did not use ^, we treat the output more liberally. - [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { - // First, we accept a symlink chain or an actual store path. - auto storePath = store->followLinksToStorePath(prefix); - // Second, we see if the store path ends in `.drv` to decide what sort - // of derived path they want. - // - // This handling predates the `^` syntax. The `^*` in - // `/nix/store/hash-foo.drv^*` unambiguously means "do the - // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could - // also unambiguously mean "do the DerivedPath::Opaque` case". - // - // Issue #7261 tracks reconsidering this `.drv` dispatching. - return storePath.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = std::move(storePath), - .outputs = OutputsSpec::All {}, - } - : (DerivedPath) DerivedPath::Opaque { - .path = std::move(storePath), - }; - }, - // If the user did use ^, we just do exactly what is written. - [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { - return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), - .outputs = outputSpec, - }; - }, - }, extendedOutputsSpec.raw()); - result.push_back(std::make_shared<InstallableStorePath>(store, std::move(derivedPath))); + result.push_back(std::make_shared<InstallableDerivedPath>( + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); continue; } catch (BadStorePath &) { } catch (...) { @@ -1081,8 +705,8 @@ void InstallablesCommand::prepare() installables = load(); } -Installables InstallablesCommand::load() { - Installables installables; +Installables InstallablesCommand::load() +{ if (_installables.empty() && useDefaultInstallables()) // FIXME: commands like "nix profile install" should not have a // default, probably. @@ -1092,16 +716,13 @@ Installables InstallablesCommand::load() { std::vector<std::string> InstallablesCommand::getFlakesForCompletion() { - if (_installables.empty()) { - if (useDefaultInstallables()) - return {"."}; - return {}; - } + if (_installables.empty() && useDefaultInstallables()) + return {"."}; return _installables; } -InstallableCommand::InstallableCommand(bool supportReadOnlyMode) - : SourceExprCommand(supportReadOnlyMode) +InstallableCommand::InstallableCommand() + : SourceExprCommand() { expectArgs({ .label = "installable", diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 3d12639b0..be77fdc81 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -103,9 +103,13 @@ struct Installable return {}; } + /* Get a cursor to each value this Installable could refer to. However + if none exists, throw exception instead of returning empty vector. */ virtual std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state); + /* Get the first and most preferred cursor this Installable could refer + to, or throw an exception if none exists. */ virtual ref<eval_cache::AttrCursor> getCursor(EvalState & state); @@ -157,58 +161,4 @@ struct Installable typedef std::vector<std::shared_ptr<Installable>> Installables; -struct InstallableValue : Installable -{ - ref<EvalState> state; - - InstallableValue(ref<EvalState> state) : state(state) {} -}; - -struct InstallableFlake : InstallableValue -{ - FlakeRef flakeRef; - Strings attrPaths; - Strings prefixes; - ExtendedOutputsSpec extendedOutputsSpec; - const flake::LockFlags & lockFlags; - mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; - - InstallableFlake( - SourceExprCommand * cmd, - ref<EvalState> state, - FlakeRef && flakeRef, - std::string_view fragment, - ExtendedOutputsSpec extendedOutputsSpec, - Strings attrPaths, - Strings prefixes, - const flake::LockFlags & lockFlags); - - std::string what() const override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } - - std::vector<std::string> getActualAttrPaths(); - - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - - DerivedPathsWithInfo toDerivedPaths() override; - - std::pair<Value *, PosIdx> toValue(EvalState & state) override; - - /* Get a cursor to every attrpath in getActualAttrPaths() that - exists. */ - std::vector<ref<eval_cache::AttrCursor>> - getCursors(EvalState & state) override; - - /* Get a cursor to the first attrpath in getActualAttrPaths() that - exists, or throw an exception with suggestions if none exists. */ - ref<eval_cache::AttrCursor> getCursor(EvalState & state) override; - - std::shared_ptr<flake::LockedFlake> getLockedFlake() const; - - FlakeRef nixpkgsFlakeRef() const override; -}; - -ref<eval_cache::EvalCache> openEvalCache( - EvalState & state, - std::shared_ptr<flake::LockedFlake> lockedFlake); - } diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 152bc388d..541a7d2ba 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,7 +6,7 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix +libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in index 1761a9f41..39575f222 100644 --- a/src/libcmd/nix-cmd.pc.in +++ b/src/libcmd/nix-cmd.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixcmd -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4158439b6..e3afb1531 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -19,6 +19,8 @@ extern "C" { } #endif +#include "repl.hh" + #include "ansicolor.hh" #include "shared.hh" #include "eval.hh" @@ -31,7 +33,9 @@ extern "C" { #include "get-drvs.hh" #include "derivations.hh" #include "globals.hh" -#include "command.hh" +#include "flake/flake.hh" +#include "flake/lockfile.hh" +#include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" @@ -45,18 +49,16 @@ extern "C" { namespace nix { struct NixRepl + : AbstractNixRepl #if HAVE_BOEHMGC - : gc + , gc #endif { std::string curDir; - ref<EvalState> state; - Bindings * autoArgs; size_t debugTraceIndex; Strings loadedFiles; - typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; std::function<AnnotatedValues()> getValues; const static int envSize = 32768; @@ -69,8 +71,11 @@ struct NixRepl NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state, std::function<AnnotatedValues()> getValues); - ~NixRepl(); - void mainLoop(); + virtual ~NixRepl(); + + void mainLoop() override; + void initEnv() override; + StringSet completePrefix(const std::string & prefix); bool getLine(std::string & input, const std::string & prompt); StorePath getDerivationPath(Value & v); @@ -78,7 +83,6 @@ struct NixRepl void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); - void initEnv(); void loadFiles(); void reloadFiles(); void addAttrsToScope(Value & attrs); @@ -92,7 +96,6 @@ struct NixRepl std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); }; - std::string removeWhitespace(std::string s) { s = chomp(s); @@ -104,7 +107,7 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, std::function<NixRepl::AnnotatedValues()> getValues) - : state(state) + : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(false, state->staticBaseEnv.get())) @@ -1029,8 +1032,22 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m return str; } -void runRepl( - ref<EvalState>evalState, + +std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create( + const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + std::function<AnnotatedValues()> getValues) +{ + return std::make_unique<NixRepl>( + searchPath, + openStore(), + state, + getValues + ); +} + + +void AbstractNixRepl::runSimple( + ref<EvalState> evalState, const ValMap & extraEnv) { auto getValues = [&]()->NixRepl::AnnotatedValues{ @@ -1054,91 +1071,4 @@ void runRepl( repl->mainLoop(); } -struct CmdRepl : InstallablesCommand -{ - CmdRepl() { - evalSettings.pureEval = false; - } - - void prepare() override - { - if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { - warn("future versions of Nix will require using `--file` to load a file"); - if (this->_installables.size() > 1) - warn("more than one input file is not currently supported"); - auto filePath = this->_installables[0].data(); - file = std::optional(filePath); - _installables.front() = _installables.back(); - _installables.pop_back(); - } - installables = InstallablesCommand::load(); - } - - std::vector<std::string> files; - - Strings getDefaultFlakeAttrPaths() override - { - return {""}; - } - - bool useDefaultInstallables() override - { - return file.has_value() or expr.has_value(); - } - - bool forceImpureByDefault() override - { - return true; - } - - std::string description() override - { - return "start an interactive environment for evaluating Nix expressions"; - } - - std::string doc() override - { - return - #include "repl.md" - ; - } - - void run(ref<Store> store) override - { - auto state = getEvalState(); - auto getValues = [&]()->NixRepl::AnnotatedValues{ - auto installables = load(); - NixRepl::AnnotatedValues values; - for (auto & installable: installables){ - auto what = installable->what(); - if (file){ - auto [val, pos] = installable->toValue(*state); - auto what = installable->what(); - state->forceValue(*val, pos); - auto autoArgs = getAutoArgs(*state); - auto valPost = state->allocValue(); - state->autoCallFunction(*autoArgs, *val, *valPost); - state->forceValue(*valPost, pos); - values.push_back( {valPost, what }); - } else { - auto [val, pos] = installable->toValue(*state); - values.push_back( {val, what} ); - } - } - return values; - }; - auto repl = std::make_unique<NixRepl>( - searchPath, - openStore(), - state, - getValues - ); - repl->autoArgs = getAutoArgs(*repl->state); - repl->initEnv(); - repl->mainLoop(); - } -}; - -static auto rCmdRepl = registerCommand<CmdRepl>("repl"); - } diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh new file mode 100644 index 000000000..dfccc93e7 --- /dev/null +++ b/src/libcmd/repl.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "eval.hh" + +#if HAVE_BOEHMGC +#define GC_INCLUDE_NEW +#include <gc/gc_cpp.h> +#endif + +namespace nix { + +struct AbstractNixRepl +{ + ref<EvalState> state; + Bindings * autoArgs; + + AbstractNixRepl(ref<EvalState> state) + : state(state) + { } + + virtual ~AbstractNixRepl() + { } + + typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; + + static std::unique_ptr<AbstractNixRepl> create( + const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + std::function<AnnotatedValues()> getValues); + + static void runSimple( + ref<EvalState> evalState, + const ValMap & extraEnv); + + virtual void initEnv() = 0; + + virtual void mainLoop() = 0; +}; + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 1828b8c2e..21fc4d0fe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2490,7 +2490,7 @@ Strings EvalSettings::getDefaultNixPath() }; if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(getHome() + "/.nix-defexpr/channels"); + add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); add(settings.nixStateDir + "/profiles/per-user/root/channels"); } diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 8061db3df..4beb0b0fe 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -16,7 +16,9 @@ let subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; - flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir); + + flake = import (outPath + "/flake.nix"); inputs = builtins.mapAttrs (inputName: inputSpec: allNodes.${resolveInput inputSpec}) @@ -43,7 +45,21 @@ let outputs = flake.outputs (inputs // { self = result; }); - result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; }; + result = + outputs + # We add the sourceInfo attribute for its metadata, as they are + # relevant metadata for the flake. However, the outPath of the + # sourceInfo does not necessarily match the outPath of the flake, + # as the flake may be in a subdirectory of a source. + # This is shadowed in the next // + // sourceInfo + // { + # This shadows the sourceInfo.outPath + inherit outPath; + + inherit inputs; inherit outputs; inherit sourceInfo; _type = "flake"; + }; + in if node.flake or true then assert builtins.isFunction flake.outputs; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a3ed90e1f..a74e68c9c 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -219,7 +219,7 @@ std::optional<FlakeRef> LockFile::isUnlocked() const visit(root); for (auto & i : nodes) { - if (i == root) continue; + if (i == ref<const Node>(root)) continue; auto node = i.dynamic_pointer_cast<const LockedNode>(); if (node && !node->lockedRef.input.isLocked()) return node->lockedRef; diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 80f7a492b..60ffb5dba 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index ffe67f97d..4a81eaa47 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -186,7 +186,7 @@ struct ExprString : Expr { std::string s; Value v; - ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; + ExprString(std::string &&s) : s(std::move(s)) { v.mkString(this->s.data()); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -233,7 +233,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; + ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -243,7 +243,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index ffb364a90..dec5818fc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -90,7 +90,7 @@ static void dupAttr(const EvalState & state, Symbol attr, const PosIdx pos, cons } -static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, +static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos, const nix::EvalState & state) { AttrPath::iterator i; @@ -188,7 +188,7 @@ static Formals * toFormals(ParseData & data, ParserFormals * formals, static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, - std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> & es) + std::vector<std::pair<PosIdx, std::variant<Expr *, StringToken>>> && es) { if (es.empty()) return new ExprString(""); @@ -268,7 +268,7 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(s2)); + es2->emplace_back(i->first, new ExprString(std::move(s2))); }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); @@ -408,7 +408,7 @@ expr_op | expr_op OR expr_op { $$ = new ExprOpOr(makeCurPos(@2, data), $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(makeCurPos(@2, data), $1, $3); } | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(makeCurPos(@2, data), $1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } | expr_op '+' expr_op { $$ = new ExprConcatStrings(makeCurPos(@2, data), false, new std::vector<std::pair<PosIdx, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } | expr_op '-' expr_op { $$ = new ExprCall(makeCurPos(@2, data), new ExprVar(data->symbols.create("__sub")), {$1, $3}); } @@ -431,14 +431,14 @@ expr_app expr_select : expr_simple '.' attrpath - { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; } | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } - | expr_simple { $$ = $1; } + | expr_simple ; expr_simple @@ -453,9 +453,10 @@ expr_simple | FLOAT { $$ = new ExprFloat($1); } | '"' string_parts '"' { $$ = $2; } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { - $$ = stripIndentation(CUR_POS, data->symbols, *$2); + $$ = stripIndentation(CUR_POS, data->symbols, std::move(*$2)); + delete $2; } - | path_start PATH_END { $$ = $1; } + | path_start PATH_END | path_start string_parts_interpolated PATH_END { $2->insert($2->begin(), {makeCurPos(@1, data), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); @@ -465,7 +466,7 @@ expr_simple $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__findFile")), {new ExprVar(data->symbols.create("__nixPath")), - new ExprString(path)}); + new ExprString(std::move(path))}); } | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); @@ -533,7 +534,7 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data), data->state); } + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, std::move(*$2), $4, makeCurPos(@2, data), data->state); delete $2; } | binds INHERIT attrs ';' { $$ = $1; for (auto & i : *$3) { @@ -542,6 +543,7 @@ binds auto pos = makeCurPos(@3, data); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } + delete $3; } | binds INHERIT '(' expr ')' attrs ';' { $$ = $1; @@ -551,6 +553,7 @@ binds dupAttr(data->state, i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } + delete $6; } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; @@ -596,7 +599,7 @@ attrpath ; attr - : ID { $$ = $1; } + : ID | OR_KW { $$ = {"or", 2}; } ; @@ -612,9 +615,9 @@ expr_list formals : formal ',' formals - { $$ = $3; $$->formals.push_back(*$1); } + { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } | formal - { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; } + { $$ = new ParserFormals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } | { $$ = new ParserFormals; $$->ellipsis = false; } | ELLIPSIS diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index db33749a1..f4ae3b537 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2093,7 +2093,7 @@ static void addPath( std::optional<StorePath> expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = *expectedHash, }, @@ -3038,9 +3038,9 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` - evaluates to 6. + ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. + The return value of each application of `op` is evaluated immediately, + even for intermediate values. )", .fun = prim_foldlStrict, }); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e194462e4..f3dce2214 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -4,6 +4,7 @@ #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "url.hh" #include <ctime> #include <iomanip> @@ -68,7 +69,16 @@ void emitTreeAttrs( std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; + if (uri.find("://") == std::string::npos) { + const auto p = ParsedURL { + .scheme = defaultScheme, + .authority = "", + .path = uri + }; + return p.to_string(); + } else { + return uri; + } } std::string fixURIForGit(std::string uri, EvalState & state) @@ -236,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - { + .hash = { .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, .hash = *expectedHash, }, @@ -461,6 +471,17 @@ static RegisterPrimOp primop_fetchGit({ > **Note** > > This behavior is disabled in *Pure evaluation mode*. + + - To fetch the content of a checked-out work directory: + + ```nix + builtins.fetchGit ./work-dir + ``` + + If the URL points to a local directory, and no `ref` or `rev` is + given, `fetchGit` will use the current content of the checked-out + files, even if they are not committed or added to Git's index. It will + only consider files added to the Git repository, as listed by `git ls-files`. )", .fun = prim_fetchGit, }); diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 5e2213f69..24e95ac39 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -1,7 +1,7 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index f1ea1b197..411bc0ac3 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" #include "value-to-json.hh" namespace nix { diff --git a/src/libexpr/tests/libexprtests.hh b/src/libexpr/tests/libexpr.hh index 03e468fbb..8534d9567 100644 --- a/src/libexpr/tests/libexprtests.hh +++ b/src/libexpr/tests/libexpr.hh @@ -7,18 +7,19 @@ #include "eval-inline.hh" #include "store-api.hh" +#include "tests/libstore.hh" namespace nix { - class LibExprTest : public ::testing::Test { + class LibExprTest : public LibStoreTest { public: static void SetUpTestSuite() { - initLibStore(); + LibStoreTest::SetUpTestSuite(); initGC(); } protected: LibExprTest() - : store(openStore("dummy://")) + : LibStoreTest() , state({}, store) { } @@ -36,7 +37,6 @@ namespace nix { return state.symbols.create(value); } - ref<Store> store; EvalState state; }; diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index e483575a4..3e5504f71 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -2,6 +2,8 @@ check: libexpr-tests_RUN programs += libexpr-tests +libexpr-tests_NAME := libnixexpr-tests + libexpr-tests_DIR := $(d) libexpr-tests_INSTALL_DIR := @@ -12,6 +14,6 @@ libexpr-tests_SOURCES := \ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -libexpr-tests_LIBS = libexpr libutil libstore libfetchers +libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index 9cdcf64a1..e1d3ac503 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -1,7 +1,7 @@ #include <gmock/gmock.h> #include <gtest/gtest.h> -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { class CaptureLogger : public Logger diff --git a/src/libexpr/tests/trivial.cc b/src/libexpr/tests/trivial.cc index 8ce276e52..171727ac7 100644 --- a/src/libexpr/tests/trivial.cc +++ b/src/libexpr/tests/trivial.cc @@ -1,4 +1,4 @@ -#include "libexprtests.hh" +#include "tests/libexpr.hh" namespace nix { // Testing of trivial expressions diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index d5c9d3bce..083359b7a 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -1,6 +1,10 @@ -#include "value/context.hh" +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> -#include "libexprtests.hh" +#include "tests/path.hh" +#include "tests/libexpr.hh" +#include "tests/value/context.hh" namespace nix { @@ -70,3 +74,54 @@ TEST_F(NixStringContextElemTest, built) { } } + +namespace rc { +using namespace nix; + +Gen<NixStringContextElem::Opaque> Arbitrary<NixStringContextElem::Opaque>::arbitrary() +{ + return gen::just(NixStringContextElem::Opaque { + .path = *gen::arbitrary<StorePath>(), + }); +} + +Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary<StorePath>(), + }); +} + +Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitrary() +{ + return gen::just(NixStringContextElem::Built { + .drvPath = *gen::arbitrary<StorePath>(), + .output = (*gen::arbitrary<StorePathName>()).name, + }); +} + +Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 2)) { + case 0: + return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>()); + case 1: + return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>()); + default: + return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>()); + } +} + +} + +namespace nix { + +RC_GTEST_FIXTURE_PROP( + NixStringContextElemTest, + prop_round_rip, + (const NixStringContextElem & o)) +{ + RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); +} + +} diff --git a/src/libexpr/tests/value/context.hh b/src/libexpr/tests/value/context.hh new file mode 100644 index 000000000..54d21760e --- /dev/null +++ b/src/libexpr/tests/value/context.hh @@ -0,0 +1,30 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <value/context.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<NixStringContextElem::Opaque> { + static Gen<NixStringContextElem::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<NixStringContextElem::Built> { + static Gen<NixStringContextElem::Built> arbitrary(); +}; + +template<> +struct Arbitrary<NixStringContextElem::DrvDeep> { + static Gen<NixStringContextElem::DrvDeep> arbitrary(); +}; + +template<> +struct Arbitrary<NixStringContextElem> { + static Gen<NixStringContextElem> arbitrary(); +}; + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index d8008c436..721563cba 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,9 +1,10 @@ #pragma once #include "util.hh" +#include "comparator.hh" #include "path.hh" -#include <optional> +#include <variant> #include <nlohmann/json_fwd.hpp> @@ -31,7 +32,9 @@ class Store; Encoded as just the path: ‘<path>’. */ struct NixStringContextElem_Opaque { - StorePath path; + StorePath path; + + GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; /* Path to a derivation and its entire build closure. @@ -43,7 +46,9 @@ struct NixStringContextElem_Opaque { Encoded in the form ‘=<drvPath>’. */ struct NixStringContextElem_DrvDeep { - StorePath drvPath; + StorePath drvPath; + + GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; /* Derivation output. @@ -51,8 +56,10 @@ struct NixStringContextElem_DrvDeep { Encoded in the form ‘!<output>!<drvPath>’. */ struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; + StorePath drvPath; + std::string output; + + GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); }; using _NixStringContextElem_Raw = std::variant< diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index f33cbdcfc..7049dea30 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -57,7 +57,7 @@ struct FetchSettings : public Config ``` This example specifies three tokens, one each for accessing - github.com, gitlab.mycompany.com, and sourceforge.net. + github.com, gitlab.mycompany.com, and gitlab.com. The `input.foo` uses the "gitlab" fetcher, which might requires specifying the token type along with the token diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 3936eadfe..91db3a9eb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -211,7 +211,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = *narHash, }, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1f7d7c07d..309a143f5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -615,15 +615,42 @@ struct GitInputScheme : InputScheme AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + { + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + } runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + /* Ensure that we use the correct origin for fetching + submodules. This matters for submodules with relative + URLs. */ + if (isLocal) { + writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + + /* Restore the config.bare setting we may have just + copied erroneously from the user's repo. */ + runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); + } else + runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); + + /* As an optimisation, copy the modules directory of the + source repo if it exists. */ + auto modulesPath = repoDir + "/" + gitDir + "/modules"; + if (pathExists(modulesPath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); + runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); + } + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + } filter = isNotDotGitDirectory; } else { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 302046610..96fe5faca 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -74,7 +74,7 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Flat, .hash = hash, }, diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 37b03dcd4..fb3ead6fa 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9058bb8b1..9eae8d534 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,7 +309,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = nar.first, }, @@ -380,7 +380,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); getFile(narInfoFile, - {[=](std::future<std::optional<std::string>> fut) { + {[=,this](std::future<std::optional<std::string>> fut) { try { auto data = fut.get(); @@ -427,7 +427,7 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6f3048b63..85e9e16ff 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -16,6 +16,7 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "namespaces.hh" #include <regex> #include <queue> @@ -167,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -205,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -372,12 +385,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -int childEntry(void * arg) -{ - ((LocalDerivationGoal *) arg)->runChild(); - return 1; -} - #if __linux__ static void linkOrCopy(const Path & from, const Path & to) { @@ -663,7 +670,8 @@ void LocalDerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - chownToBuilder(chrootRootDir + "/etc"); + if (parsedDrv->useUidRange()) + chownToBuilder(chrootRootDir + "/etc"); if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); @@ -888,12 +896,7 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; + usingUserNamespace = userNamespacesSupported(); Pid helper = startProcess([&]() { @@ -908,76 +911,21 @@ void LocalDerivationGoal::startBuilder() if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) - flags |= CLONE_NEWNET; + options.cloneFlags |= CLONE_NEWNET; if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -1045,9 +993,6 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { runChild(); }); @@ -1516,8 +1461,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + daemon::NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -1907,6 +1851,10 @@ void LocalDerivationGoal::runChild() } } + /* Make /etc unwritable */ + if (!parsedDrv->useUidRange()) + chmod_(chrootRootDir + "/etc", 0555); + /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount namespace. This means that the call to setns() in @@ -2323,11 +2271,28 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get<bool>(); + } + } + } + + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b94fb8416..f775f8486 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -276,7 +276,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 5452758b3..034a9485b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,8 +17,9 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + default: + throw Error("impossible, caught both cases"); } - assert(false); } std::string makeContentAddressingPrefix(ContentAddressMethod m) { @@ -168,13 +169,13 @@ ContentAddressWithReferences contentAddressFromMethodHashAndRefs( if (refs.self) throw UsageError("Cannot have a self reference with text hashing scheme"); return TextInfo { - { .hash = std::move(hash) }, + .hash = { .hash = std::move(hash) }, .references = std::move(refs.others), }; }, [&](FileIngestionMethod m2) -> ContentAddressWithReferences { return FixedOutputInfo { - { + .hash = { .method = m2, .hash = std::move(hash), }, @@ -191,7 +192,7 @@ ContentAddressMethod getContentAddressMethod(const ContentAddressWithReferences return TextHashMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.method; + return fsh.hash.method; }, }, ca); } @@ -222,13 +223,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { - h, + .hash = h, .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { - h, + .hash = h, .references = {}, }; }, @@ -239,10 +240,10 @@ Hash getContentAddressHash(const ContentAddressWithReferences & ca) { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash; + return th.hash.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash; + return fsh.hash.hash; }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 729b1078d..962a50c6b 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -118,18 +118,20 @@ struct StoreReferences { */ // This matches the additional info that we need for makeTextPath -struct TextInfo : TextHash { +struct TextInfo { + TextHash hash; // References for the paths, self references disallowed StorePathSet references; - GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); + GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo : FixedOutputHash { +struct FixedOutputInfo { + FixedOutputHash hash; // References for the paths StoreReferences references; - GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; typedef std::variant< diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0694f5b90..1745b1e57 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -222,7 +222,8 @@ struct ClientSettings else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; @@ -235,6 +236,10 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString<Paths>(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name @@ -528,7 +533,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -545,7 +557,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); @@ -984,8 +998,7 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function<void(Store &)> authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr; @@ -1028,10 +1041,6 @@ void processConnection( try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..8c765615c 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -13,11 +13,6 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function<void(Store &)> authHook); + RecursiveFlag recursive); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b19ee5ed1..5461f4a9c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -6,6 +6,7 @@ #include "worker-protocol.hh" #include "fs-accessor.hh" #include <boost/container/small_vector.hpp> +#include <nlohmann/json.hpp> namespace nix { @@ -887,4 +888,64 @@ std::optional<BasicDerivation> Derivation::tryResolve( const Hash impureOutputHash = hashString(htSHA256, "impure"); +nlohmann::json DerivationOutput::toJSON( + const Store & store, std::string_view drvName, std::string_view outputName) const +{ + nlohmann::json res = nlohmann::json::object(); + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doi) { + res["path"] = store.printStorePath(doi.path); + }, + [&](const DerivationOutput::CAFixed & dof) { + res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + res["hashAlgo"] = printMethodAlgo(dof.ca); + res["hash"] = getContentAddressHash(dof.ca).to_string(Base16, false); + // FIXME print refs? + }, + [&](const DerivationOutput::CAFloating & dof) { + res["hashAlgo"] = makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType); + }, + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + res["hashAlgo"] = makeContentAddressingPrefix(doi.method) + printHashType(doi.hashType); + res["impure"] = true; + }, + }, raw()); + return res; +} + +nlohmann::json Derivation::toJSON(const Store & store) const +{ + nlohmann::json res = nlohmann::json::object(); + + { + nlohmann::json & outputsObj = res["outputs"]; + outputsObj = nlohmann::json::object(); + for (auto & [outputName, output] : outputs) { + outputsObj[outputName] = output.toJSON(store, name, outputName); + } + } + + { + auto& inputsList = res["inputSrcs"]; + inputsList = nlohmann::json ::array(); + for (auto & input : inputSrcs) + inputsList.emplace_back(store.printStorePath(input)); + } + + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json ::object(); + for (auto & input : inputDrvs) + inputDrvsObj[store.printStorePath(input.first)] = input.second; + } + + res["system"] = platform; + res["builder"] = builder; + res["args"] = args; + res["env"] = env; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 803ab3cd6..80629dadb 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -83,6 +83,11 @@ struct DerivationOutput : _DerivationOutputRaw inline const Raw & raw() const { return static_cast<const Raw &>(*this); } + + nlohmann::json toJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName) const; }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; @@ -210,6 +215,8 @@ struct Derivation : BasicDerivation Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + + nlohmann::json toJSON(const Store & store) const; }; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5f2dfa4f1..9e0cce377 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -4,8 +4,8 @@ #include "path.hh" #include "realisation.hh" #include "outputs-spec.hh" +#include "comparator.hh" -#include <optional> #include <variant> #include <nlohmann/json_fwd.hpp> @@ -28,8 +28,7 @@ struct DerivedPathOpaque { std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + GENERATE_CMP(DerivedPathOpaque, me->path); }; /** @@ -52,8 +51,7 @@ struct DerivedPathBuilt { static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref<Store> store) const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); }; using _DerivedPathRaw = std::variant< @@ -97,6 +95,8 @@ struct BuiltPathBuilt { nlohmann::json toJSON(ref<Store> store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); }; using _BuiltPathRaw = std::variant< diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 756bd4423..1c8676a59 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -101,6 +101,7 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) @@ -828,7 +829,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) { auto state(_state->lock()); - while (state->data.empty()) { + if (state->data.empty()) { if (state->quit) { if (state->exc) std::rethrow_exception(state->exc); @@ -836,6 +837,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } state.wait(state->avail); + + if (state->data.empty()) continue; } chunk = std::move(state->data); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 130c5b670..8e33a3dec 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -222,19 +222,19 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s .longName = name, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7111def92..93086eaf8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -201,7 +201,16 @@ public: {"build-timeout"}}; PathSetting buildHook{this, true, "", "build-hook", - "The path of the helper program that executes builds to remote machines."}; + R"( + The path to the helper program that executes remote builds. + + Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. + The default value is the internal Nix binary that implements remote building. + + > **Important** + > + > Change this setting only if you really know what you’re doing. + )"}; Setting<std::string> builders{ this, "@" + nixConfDir + "/machines", "builders", @@ -279,8 +288,8 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. Defaults to `nixbld` when running as root, *empty* otherwise. )", @@ -570,11 +579,15 @@ public: {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a substituter): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -670,13 +683,14 @@ public: independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. - Nix will copy a store path from a remote store only if one - of the following is true: + At least one of the following conditions must be met for Nix to use + a substituter: - - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) + - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described + in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; @@ -691,24 +705,6 @@ public: )", {"trusted-binary-caches"}}; - Setting<Strings> trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting<unsigned int> ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -731,18 +727,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting<Strings> allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting<bool> printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; @@ -970,6 +954,27 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; + + Setting<bool> useXDGBaseDirectories{ + this, false, "use-xdg-base-directories", + R"( + If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`. + The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md). + + [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + + > **Warning** + > This changes the location of some well-known symlinks that Nix creates, which might break tools that rely on the old, non-XDG-conformant locations. + + In particular, the following locations change: + + | Old | New | + |-------------------|--------------------------------| + | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | + | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | + | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + )" + }; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 73bcd6e81..1479822a9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -56,7 +56,7 @@ public: void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 94c005130..72ba12e0d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -201,8 +201,6 @@ LocalStore::LocalStore(const Params & params) throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -1417,7 +1415,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, @@ -1844,20 +1842,6 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06d36a7d5..a84eb7c26 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,8 +281,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -31,11 +31,12 @@ std::map<StorePath, StorePath> makeContentAddressed( for (auto & ref : oldInfo->references) { if (ref == path) refs.self = true; - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) { - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); refs.others.insert(std::move(replacement)); } } @@ -51,7 +52,7 @@ std::map<StorePath, StorePath> makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index acd4e0929..e16b60b88 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto & ref : referrers) + for (auto& ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto & i : queryValidDerivers(path)) + for (auto& i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,12 +40,12 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future<ref<const ValidPathInfo>> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) + for (auto& ref : info->references) if (ref != path) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); @@ -93,12 +93,12 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) [&](const TextInfo & ti) -> std::optional<ContentAddress> { if (!ti.references.empty()) return std::nullopt; - return static_cast<TextHash>(ti); + return ti.hash; }, [&](const FixedOutputInfo & fi) -> std::optional<ContentAddress> { if (!fi.references.empty()) return std::nullopt; - return static_cast<FixedOutputHash>(fi); + return fi.hash; }, }, dof->ca); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3e0689534..2645f468b 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -84,11 +84,10 @@ public: Sync<State> _state; - NarInfoDiskCacheImpl() + NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite") { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -98,7 +97,7 @@ public: state->db.exec(schema); state->insertCache.create(state->db, - "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); state->queryCache.create(state->db, "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); @@ -166,6 +165,8 @@ public: return i->second; } +private: + std::optional<Cache> queryCacheRaw(State & state, const std::string & uri) { auto i = state.caches.find(uri); @@ -173,15 +174,21 @@ public: auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl)); if (!queryCache.next()) return std::nullopt; - state.caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + auto cache = Cache { + .id = (int) queryCache.getInt(0), + .storeDir = queryCache.getStr(1), + .wantMassQuery = queryCache.getInt(2) != 0, + .priority = (int) queryCache.getInt(3), + }; + state.caches.emplace(uri, cache); } return getCache(state, uri); } - void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override +public: + int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override { - retrySQLite<void>([&]() { + return retrySQLite<int>([&]() { auto state(_state.lock()); SQLiteTxn txn(state->db); @@ -190,17 +197,29 @@ public: auto cache(queryCacheRaw(*state, uri)); if (cache) - return; + return cache->id; - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + Cache ret { + .id = -1, // set below + .storeDir = storeDir, + .wantMassQuery = wantMassQuery, + .priority = priority, + }; + + { + auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); + assert(r.next()); + ret.id = (int) r.getInt(0); + } + + state->caches[uri] = ret; txn.commit(); + return ret.id; }); } - std::optional<CacheInfo> cacheExists(const std::string & uri) override + std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override { return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> { auto state(_state.lock()); @@ -208,6 +227,7 @@ public: if (!cache) return std::nullopt; return CacheInfo { + .id = cache->id, .wantMassQuery = cache->wantMassQuery, .priority = cache->priority }; @@ -371,4 +391,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache() return cache; } +ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath) +{ + return make_ref<NarInfoDiskCacheImpl>(dbPath); +} + } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 2dcaa76a4..4877f56d8 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -13,16 +13,17 @@ public: virtual ~NarInfoDiskCache() { } - virtual void createCache(const std::string & uri, const Path & storeDir, + virtual int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; struct CacheInfo { + int id; bool wantMassQuery; int priority; }; - virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0; + virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0; virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; @@ -45,4 +46,6 @@ public: multiple threads. */ ref<NarInfoDiskCache> getNarInfoDiskCache(); +ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath); + } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 6d67b1e03..dc42d0bca 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06..76cab63e0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,7 +21,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferenences() const +std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const { if (! ca) return std::nullopt; @@ -30,7 +30,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef [&](const TextHash & th) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - th, + .hash = th, .references = references, }; }, @@ -42,7 +42,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - foh, + .hash = foh, .references = { .others = std::move(refs), .self = hasSelfReference, @@ -54,7 +54,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = contentAddressWithReferences(); if (! fullCaOpt) return false; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d94540..97eb6638b 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,6 +29,7 @@ struct ValidPathInfo { StorePath path; std::optional<StorePath> deriver; + // TODO document this Hash narHash; StorePathSet references; time_t registrationTime = 0; @@ -77,7 +78,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional<ContentAddressWithReferences> contentAddressWithReferenences() const; + std::optional<ContentAddressWithReferences> contentAddressWithReferences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..c551c5f3e 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -280,16 +280,24 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = createNixStateDir() + "/profiles"; + createDirs(profileRoot); + return profileRoot; +} + + Path getDefaultProfile() { - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { + auto profile = + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : profilesDir() + "/profile"; if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + replaceSymlink(profile, profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..fbf95b850 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,8 +68,13 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); -/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create - it. */ +/* Creates and returns the path to a directory suitable for storing the user’s + profiles. */ +Path profilesDir(); + +/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ + nix/profile if XDG Base Directory Support is enabled), and create if doesn't + exist */ Path getDefaultProfile(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5f1bb73ac..e19d512c0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 844553ad3..8d76eee99 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { - if (auto cacheInfo = diskCache->cacheExists(getUri())) { + if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 353dff9fa..871f2f3be 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -41,6 +41,15 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } +static void traceSQL(void * x, const char * sql) +{ + // wacky delimiters: + // so that we're quite unambiguous without escaping anything + // notice instead of trace: + // so that this can be enabled without getting the firehose in our face. + notice("SQL<[%1%]>", sql); +}; + SQLite::SQLite(const Path & path, bool create) { // useSQLiteWAL also indicates what virtual file system we need. Using @@ -58,6 +67,11 @@ SQLite::SQLite(const Path & path, bool create) if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) SQLiteError::throw_(db, "setting timeout"); + if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") { + // To debug sqlite statements; trace all of them + sqlite3_trace(db, &traceSQL, nullptr); + } + exec("pragma foreign_keys = 1"); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706..b8a77b324 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -99,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). <id> is the name of the output (usually, "out"). - <h2> = base-16 representation of a SHA-256 hash of: + <h2> = base-16 representation of a SHA-256 hash of <s2> + + <s2> = if <type> = "text:...": the string written to the resulting store path - if <type> = "source": + if <type> = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if <type> = "output:<id>": @@ -164,8 +166,8 @@ StorePath Store::makeOutputPath(std::string_view id, /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ + hacky, but we can't put them in, say, <s2> (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, @@ -182,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } @@ -198,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash, + info.hash.hash, name); } @@ -230,7 +232,7 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - { + .hash = { .method = method, .hash = h, }, @@ -439,7 +441,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, @@ -762,7 +764,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); @@ -996,7 +998,7 @@ void copyStorePath( auto info2 = make_ref<ValidPathInfo>(*info); info2->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), - info->contentAddressWithReferenences().value()); + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1112,7 @@ std::map<StorePath, StorePath> copyPaths( if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2d252db84..72517c6e4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -653,9 +653,6 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - /* * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc new file mode 100644 index 000000000..4dd14dcce --- /dev/null +++ b/src/libstore/tests/derivation.cc @@ -0,0 +1,143 @@ +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> + +#include "derivations.hh" + +#include "tests/libstore.hh" + +namespace nix { + +class DerivationTest : public LibStoreTest +{ +}; + +#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ + TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ + } + +TEST_JSON(DerivationOutput, inputAddressed, + R"({ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFixed, + R"({ + "hashAlgo": "r:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + .references = {}, + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFixedText, + R"({ + "hashAlgo": "text:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = TextInfo { + .hash = { + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + .references = {}, + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFloating, + R"({ + "hashAlgo": "r:sha256" + })", + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, deferred, + R"({ })", + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, impure, + R"({ + "hashAlgo": "r:sha256", + "impure": true + })", + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(Derivation, impure, + R"({ + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ + "cat", + "dog" + ] + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + ({ + Derivation drv; + drv.inputSrcs = { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + } + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + drv; + })) + +#undef TEST_JSON + +} diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc new file mode 100644 index 000000000..d1ac2c5e7 --- /dev/null +++ b/src/libstore/tests/derived-path.cc @@ -0,0 +1,62 @@ +#include <regex> + +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace rc { +using namespace nix; + +Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary<StorePath>(), + }); +} + +Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = *gen::arbitrary<StorePath>(), + .outputs = *gen::arbitrary<OutputsSpec>(), + }); +} + +Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 1)) { + case 0: + return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>()); + default: + return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>()); + } +} + +} + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +} diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh new file mode 100644 index 000000000..3bc812440 --- /dev/null +++ b/src/libstore/tests/derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <derived-path.hh> + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<DerivedPath::Opaque> { + static Gen<DerivedPath::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<DerivedPath::Built> { + static Gen<DerivedPath::Built> arbitrary(); +}; + +template<> +struct Arbitrary<DerivedPath> { + static Gen<DerivedPath> arbitrary(); +}; + +} diff --git a/src/libstore/tests/libstoretests.hh b/src/libstore/tests/libstore.hh index 05397659b..05397659b 100644 --- a/src/libstore/tests/libstoretests.hh +++ b/src/libstore/tests/libstore.hh diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index a2cf8a0cf..03becc7d1 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -1,6 +1,20 @@ -check: libstore-tests_RUN +check: libstore-tests-exe_RUN -programs += libstore-tests +programs += libstore-tests-exe + +libstore-tests-exe_NAME = libnixstore-tests + +libstore-tests-exe_DIR := $(d) + +libstore-tests-exe_INSTALL_DIR := + +libstore-tests-exe_LIBS = libstore-tests + +libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libstore-tests + +libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) @@ -10,6 +24,6 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil -libstore-tests_LIBS = libstore libutil +libstore-tests_LIBS = libutil-tests libstore libutil libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/src/libstore/tests/nar-info-disk-cache.cc new file mode 100644 index 000000000..b4bdb8329 --- /dev/null +++ b/src/libstore/tests/nar-info-disk-cache.cc @@ -0,0 +1,123 @@ +#include "nar-info-disk-cache.hh" + +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> +#include "sqlite.hh" +#include <sqlite3.h> + + +namespace nix { + +TEST(NarInfoDiskCacheImpl, create_and_read) { + // This is a large single test to avoid some setup overhead. + + int prio = 12345; + bool wantMassQuery = true; + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir); + Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite"); + + int savedId; + int barId; + SQLite db; + SQLiteStmt getIds; + + { + auto cache = getTestNarInfoDiskCache(dbPath); + + // Set up "background noise" and check that different caches receive different ids + { + auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + ASSERT_NE(bc1, bc2); + barId = bc1; + } + + // Check that the fields are saved and returned correctly. This does not test + // the select statement yet, because of in-memory caching. + savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);; + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(savedId, r->id); + } + + // We're going to pay special attention to the id field because we had a bug + // that changed it. + db = SQLite(dbPath); + getIds.create(db, "select id from BinaryCaches where url = 'http://foo'"); + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + ASSERT_EQ(savedId, q.getInt(0)); + ASSERT_FALSE(q.next()); + } + + // Pretend that the caches are older, but keep one up to date, as "background noise" + db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';"); + + // This shows that the in-memory cache works + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + } + } + + { + // We can't clear the in-memory cache, so we use a new cache object. This is + // more realistic anyway. + auto cache2 = getTestNarInfoDiskCache(dbPath); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_FALSE(r); + } + + // "Update", same data, check that the id number is reused + cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->id, savedId); + } + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + auto currentId = q.getInt(0); + ASSERT_FALSE(q.next()); + ASSERT_EQ(currentId, savedId); + } + + // Check that the fields can be modified, and the id remains the same + { + auto r0 = cache2->upToDateCacheExists("https://bar"); + ASSERT_FALSE(r0); + + cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + auto r = cache2->upToDateCacheExists("https://bar"); + ASSERT_EQ(r->wantMassQuery, !wantMassQuery); + ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->id, barId); + } + + // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp) + // { + // cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20); + // auto r = cache2->upToDateCacheExists("https://bar"); + // ASSERT_EQ(r->wantMassQuery, wantMassQuery); + // ASSERT_EQ(r->priority, prio + 20); + // } + } +} + +} diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index 06e4cabbd..984d1d963 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -2,6 +2,7 @@ #include <nlohmann/json.hpp> #include <gtest/gtest.h> +#include <rapidcheck/gtest.h> namespace nix { @@ -199,3 +200,34 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON } + +namespace rc { +using namespace nix; + +Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 1)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + default: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container<StringSet>(gen::map( + gen::arbitrary<StorePathName>(), + [](StorePathName n) { return n.name; }))), + }); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh new file mode 100644 index 000000000..2d455c817 --- /dev/null +++ b/src/libstore/tests/outputs-spec.hh @@ -0,0 +1,17 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <outputs-spec.hh> + +#include <tests/path.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<OutputsSpec> { + static Gen<OutputsSpec> arbitrary(); +}; + +} diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 8ea252c92..430aa0099 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -7,7 +7,9 @@ #include "path-regex.hh" #include "store-api.hh" -#include "libstoretests.hh" +#include "tests/hash.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" namespace nix { @@ -73,17 +75,14 @@ void showValue(const StorePath & p, std::ostream & os) { namespace rc { using namespace nix; -template<> -struct Arbitrary<StorePath> { - static Gen<StorePath> arbitrary(); -}; - -Gen<StorePath> Arbitrary<StorePath>::arbitrary() +Gen<StorePathName> Arbitrary<StorePathName>::arbitrary() { - auto len = *gen::inRange<size_t>(1, StorePath::MaxPathLen); + auto len = *gen::inRange<size_t>( + 1, + StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - std::string pre { HASH_PART "-" }; - pre.reserve(pre.size() + len); + std::string pre; + pre.reserve(len); for (size_t c = 0; c < len; ++c) { switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) { @@ -118,7 +117,17 @@ Gen<StorePath> Arbitrary<StorePath>::arbitrary() } } - return gen::just(StorePath { pre }); + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen<StorePath> Arbitrary<StorePath>::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary<Hash>(), + (*gen::arbitrary<StorePathName>()).name, + }); } } // namespace rc diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh new file mode 100644 index 000000000..d7f1a8988 --- /dev/null +++ b/src/libstore/tests/path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <path.hh> + +namespace nix { + +struct StorePathName { + std::string name; +}; + +} + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<StorePathName> { + static Gen<StorePathName> arbitrary(); +}; + +template<> +struct Arbitrary<StorePath> { + static Gen<StorePath> arbitrary(); +}; + +} diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 753980fd4..35686a8aa 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -29,7 +29,15 @@ void Args::removeFlag(const std::string & longName) void Completions::add(std::string completion, std::string description) { - assert(description.find('\n') == std::string::npos); + description = trim(description); + // ellipsize overflowing content on the back of the description + auto end_index = description.find_first_of(".\n"); + if (end_index != std::string::npos) { + auto needs_ellipsis = end_index != description.size() - 1; + description.resize(end_index); + if (needs_ellipsis) + description.append(" [...]"); + } insert(Completion { .completion = completion, .description = description @@ -324,7 +332,7 @@ MultiCommand::MultiCommand(const Commands & commands_) expectArgs({ .label = "subcommand", .optional = true, - .handler = {[=](std::string s) { + .handler = {[=,this](std::string s) { assert(!command); auto i = commands.find(s); if (i == commands.end()) { diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 9bb412b4f..b349f2d80 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -209,7 +209,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s); }}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, }); if (isAppendable()) @@ -218,7 +218,7 @@ void BaseSetting<T>::convertToArg(Args & args, const std::string & category) .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, - .handler = {[=](std::string s) { overridden = true; set(s, true); }}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, }); } @@ -270,13 +270,13 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(true); }} + .handler = {[this]() { override(true); }} }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[=]() { override(false); }} + .handler = {[this]() { override(false); }} }); } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 79ec0f9cf..7ac43c854 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -250,11 +250,15 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } - bool operator ==(const T & v2) const { return value == v2; } - bool operator !=(const T & v2) const { return value != v2; } - void operator =(const T & v) { assign(v); } + template<typename U> + bool operator ==(const U & v2) const { return value == v2; } + template<typename U> + bool operator !=(const U & v2) const { return value != v2; } + template<typename U> + void operator =(const U & v) { assign(v); } virtual void assign(const T & v) { value = v; } - void setDefault(const T & v) { if (!overridden) value = v; } + template<typename U> + void setDefault(const U & v) { if (!overridden) value = v; } void set(const std::string & str, bool append = false) override; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 7d236028c..0ebeaba61 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -74,6 +74,8 @@ struct AbstractPos virtual void print(std::ostream & out) const = 0; std::optional<LinesOfCode> getCodeLines() const; + + virtual ~AbstractPos() = default; }; std::ostream & operator << (std::ostream & str, const AbstractPos & pos); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e0902971e..58d762ebb 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,6 +16,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, { Xp::Cgroups, "cgroups" }, + { Xp::DiscardReferences, "discard-references" }, }; const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index af775feb0..ac372e03e 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -25,6 +25,7 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, + DiscardReferences, }; /** diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc new file mode 100644 index 000000000..f66accb10 --- /dev/null +++ b/src/libutil/namespaces.cc @@ -0,0 +1,97 @@ +#if __linux__ + +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" + +#include <sys/mount.h> + +namespace nix { + +bool userNamespacesSupported() +{ + static auto res = [&]() -> bool + { + if (!pathExists("/proc/self/ns/user")) { + debug("'/proc/self/ns/user' does not exist; your kernel was likely built without CONFIG_USER_NS=y"); + return false; + } + + Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; + if (!pathExists(maxUserNamespaces) || + trim(readFile(maxUserNamespaces)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/user/max_user_namespaces'"); + return false; + } + + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") + { + debug("user namespaces appear to be disabled; check '/proc/sys/kernel/unprivileged_userns_clone'"); + return false; + } + + try { + Pid pid = startProcess([&]() + { + _exit(0); + }, { + .cloneFlags = CLONE_NEWUSER + }); + + auto r = pid.wait(); + assert(!r); + } catch (SysError & e) { + debug("user namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +bool mountAndPidNamespacesSupported() +{ + static auto res = [&]() -> bool + { + try { + + Pid pid = startProcess([&]() + { + /* Make sure we don't remount the parent's /proc. */ + if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) + _exit(1); + + /* Test whether we can remount /proc. The kernel disallows + this if /proc is not fully visible, i.e. if there are + filesystems mounted on top of files inside /proc. See + https://lore.kernel.org/lkml/87tvsrjai0.fsf@xmission.com/T/. */ + if (mount("none", "/proc", "proc", 0, 0) == -1) + _exit(2); + + _exit(0); + }, { + .cloneFlags = CLONE_NEWNS | CLONE_NEWPID | (userNamespacesSupported() ? CLONE_NEWUSER : 0) + }); + + if (pid.wait()) { + debug("PID namespaces do not work on this system: cannot remount /proc"); + return false; + } + + } catch (SysError & e) { + debug("mount namespaces do not work on this system: %s", e.msg()); + return false; + } + + return true; + }(); + return res; +} + +} + +#endif diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh new file mode 100644 index 000000000..e82379b9c --- /dev/null +++ b/src/libutil/namespaces.hh @@ -0,0 +1,13 @@ +#pragma once + +namespace nix { + +#if __linux__ + +bool userNamespacesSupported(); + +bool mountAndPidNamespacesSupported(); + +#endif + +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 412c03030..e4e928b3b 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -1,5 +1,12 @@ -#include "hash.hh" +#include <regex> + +#include <nlohmann/json.hpp> #include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include <hash.hh> + +#include "tests/hash.hh" namespace nix { @@ -73,3 +80,16 @@ namespace nix { "c7d329eeb6dd26545e96e55b874be909"); } } + +namespace rc { +using namespace nix; + +Gen<Hash> Arbitrary<Hash>::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary<uint8_t>(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/src/libutil/tests/hash.hh new file mode 100644 index 000000000..9e9650e6e --- /dev/null +++ b/src/libutil/tests/hash.hh @@ -0,0 +1,15 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <hash.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<Hash> { + static Gen<Hash> arbitrary(); +}; + +} diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 815e18560..167915439 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -2,14 +2,28 @@ check: libutil-tests_RUN programs += libutil-tests +libutil-tests-exe_NAME = libnixutil-tests + +libutil-tests-exe_DIR := $(d) + +libutil-tests-exe_INSTALL_DIR := + +libutil-tests-exe_LIBS = libutil-tests + +libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libutil-tests + +libutil-tests_NAME = libnixutil-tests + libutil-tests_DIR := $(d) libutil-tests_INSTALL_DIR := libutil-tests_SOURCES := $(wildcard $(d)/*.cc) -libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr +libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil -libutil-tests_LDFLAGS := $(GTEST_LIBS) +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index c3b233797..a908631e6 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -99,6 +99,27 @@ namespace nix { ASSERT_EQ(parsed, expected); } + TEST(parseURL, parsesFilePlusHttpsUrl) { + auto s = "file+https://www.example.org/video.mp4"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "file+https://www.example.org/video.mp4", + .base = "https://www.example.org/video.mp4", + .scheme = "file+https", + .authority = "www.example.org", + .path = "/video.mp4", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { + auto s = "file://www.example.org/video.mp4"; + ASSERT_THROW(parseURL(s), Error); + } TEST(parseURL, parseIPv4Address) { auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; @@ -281,4 +302,37 @@ namespace nix { ASSERT_EQ(d, s); } + + /* ---------------------------------------------------------------------------- + * percentEncode + * --------------------------------------------------------------------------*/ + + TEST(percentEncode, encodesUrlEncodedString) { + std::string s = percentEncode("==@=="); + std::string d = "%3D%3D%40%3D%3D"; + ASSERT_EQ(d, s); + } + + TEST(percentEncode, keepArgument) { + std::string a = percentEncode("abd / def"); + std::string b = percentEncode("abd / def", "/"); + ASSERT_EQ(a, "abd%20%2F%20def"); + ASSERT_EQ(b, "abd%20/%20def"); + } + + TEST(percentEncode, inverseOfDecode) { + std::string original = "%3D%3D%40%3D%3D"; + std::string once = percentEncode(original); + std::string back = percentDecode(once); + + ASSERT_EQ(back, original); + } + + TEST(percentEncode, trailingPercent) { + std::string s = percentEncode("==@==%"); + std::string d = "%3D%3D%40%3D%3D%25"; + + ASSERT_EQ(d, s); + } + } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 5b7abeb49..9e44241ac 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -30,13 +30,13 @@ ParsedURL parseURL(const std::string & url) auto & query = match[6]; auto & fragment = match[7]; - auto isFile = scheme.find("file") != std::string::npos; + auto transportIsFile = parseUrlScheme(scheme).transport == "file"; - if (authority && *authority != "" && isFile) + if (authority && *authority != "" && transportIsFile) throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); - if (isFile && path.empty()) + if (transportIsFile && path.empty()) path = "/"; return ParsedURL{ @@ -88,17 +88,22 @@ std::map<std::string, std::string> decodeQuery(const std::string & query) return result; } -std::string percentEncode(std::string_view s) +const static std::string allowedInQuery = ":@/?"; +const static std::string allowedInPath = ":@/"; + +std::string percentEncode(std::string_view s, std::string_view keep) { std::string res; for (auto & c : s) + // unreserved + keep if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') - || strchr("-._~!$&'()*+,;=:@", c)) + || strchr("-._~", c) + || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02x", (unsigned int) c); + res += fmt("%%%02X", (unsigned int) c); return res; } @@ -109,9 +114,9 @@ std::string encodeQuery(const std::map<std::string, std::string> & ss) for (auto & [name, value] : ss) { if (!first) res += '&'; first = false; - res += percentEncode(name); + res += percentEncode(name, allowedInQuery); res += '='; - res += percentEncode(value); + res += percentEncode(value, allowedInQuery); } return res; } @@ -122,7 +127,7 @@ std::string ParsedURL::to_string() const scheme + ":" + (authority ? "//" + *authority : "") - + path + + percentEncode(path, allowedInPath) + (query.empty() ? "" : "?" + encodeQuery(query)) + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 2a9fb34c1..ddd673d65 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -22,6 +22,7 @@ struct ParsedURL MakeError(BadURL, Error); std::string percentDecode(std::string_view in); +std::string percentEncode(std::string_view s, std::string_view keep=""); std::map<std::string, std::string> decodeQuery(const std::string & query); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 993dc1cb6..885bae69c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -36,6 +36,7 @@ #ifdef __linux__ #include <sys/prctl.h> #include <sys/resource.h> +#include <sys/mman.h> #include <cmath> #endif @@ -537,6 +538,16 @@ std::string getUserName() return name; } +Path getHomeOf(uid_t userId) +{ + std::vector<char> buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} Path getHome() { @@ -558,13 +569,7 @@ Path getHome() } } if (!homeDir) { - std::vector<char> buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - homeDir = pw->pw_dir; + homeDir = getHomeOf(geteuid()); if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); } @@ -604,6 +609,19 @@ Path getDataDir() return dataDir ? *dataDir : getHome() + "/.local/share"; } +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + std::optional<Path> getSelfExe() { @@ -1047,9 +1065,17 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) } +static int childEntry(void * arg) +{ + auto main = (std::function<void()> *) arg; + (*main)(); + return 1; +} + + pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { - auto wrapper = [&]() { + std::function<void()> wrapper = [&]() { if (!options.allowVfork) logger = makeSimpleLogger(); try { @@ -1069,7 +1095,27 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) _exit(1); }; - pid_t pid = doFork(options.allowVfork, wrapper); + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); return pid; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9b149de80..b5625ecef 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -137,6 +137,9 @@ void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); +/* Return the given user's home directory from /etc/passwd. */ +Path getHomeOf(uid_t userId); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); @@ -155,6 +158,12 @@ Path getDataDir(); /* Return the path of the current executable. */ std::optional<Path> getSelfExe(); +/* Return $XDG_STATE_HOME or $HOME/.local/state. */ +Path getStateDir(); + +/* Create the Nix state directory and return the path to it. */ +Path createNixStateDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -298,6 +307,7 @@ struct ProcessOptions bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; + int cloneFlags = 0; // use clone() with the specified flags (Linux only) }; pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 049838bb1..da76c2ace 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -536,7 +536,9 @@ static void main_nix_build(int argc, char * * argv) "SHELL=%5%; " "BASH=%5%; " "set +e; " - R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + + (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index cf52b03b4..338a7d18e 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ +#include "profiles.hh" #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "util.hh" #include <fcntl.h> #include <regex> @@ -162,11 +164,11 @@ static int main_nix_channel(int argc, char ** argv) { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); - channelsList = home + "/.nix-channels"; - nixDefExpr = home + "/.nix-defexpr"; + channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; + nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; // Figure out the name of the channels profile. - profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + profile = profilesDir() + "/channels"; enum { cNone, diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 406e548c0..0daf374de 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1289,7 +1289,7 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; switchLink(profileLink, profile); } @@ -1393,7 +1393,10 @@ static int main_nix_env(int argc, char * * argv) Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; + { + Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + globals.instSource.nixExprPath = nixExprPath; + } globals.instSource.systemFilter = "*"; if (!pathExists(globals.instSource.nixExprPath)) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5b261ecc6..735d6a592 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -216,7 +216,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - { + .hash = { .method = method, .hash = Hash::parseAny(hash, hashAlgo), }, diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 81dbc09a6..16e48a39b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,7 +45,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - { + .hash = { .method = std::move(ingestionMethod), .hash = std::move(hash), }, diff --git a/src/nix/app.cc b/src/nix/app.cc index 08cd0ccd4..5cd65136f 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -1,4 +1,5 @@ #include "installables.hh" +#include "installable-derived-path.hh" #include "store-api.hh" #include "eval-inline.hh" #include "eval-cache.hh" @@ -8,30 +9,6 @@ namespace nix { -struct InstallableDerivedPath : Installable -{ - ref<Store> store; - const DerivedPath derivedPath; - - InstallableDerivedPath(ref<Store> store, const DerivedPath & derivedPath) - : store(store) - , derivedPath(derivedPath) - { - } - - std::string what() const override { return derivedPath.to_string(*store); } - - DerivedPathsWithInfo toDerivedPaths() override - { - return {{derivedPath}}; - } - - std::optional<StorePath> getStorePath() override - { - return std::nullopt; - } -}; - /** * Return the rewrites that are needed to resolve a string whose context is * included in `dependencies`. @@ -146,7 +123,7 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store) for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared<InstallableDerivedPath>(store, ctxElt)); + std::make_shared<InstallableDerivedPath>(store, DerivedPath { ctxElt })); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 6ae9460f6..dcf9a6f2d 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index c527fdb0a..a22bccba1 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -34,6 +34,43 @@ using namespace nix; using namespace nix::daemon; +struct AuthorizationSettings : Config { + + Setting<Strings> trustedUsers{ + this, {"root"}, "trusted-users", + R"( + A list of names of users (separated by whitespace) that have + additional rights when connecting to the Nix daemon, such as the + ability to specify additional binary caches, or to import unsigned + NARs. You can also specify groups by prefixing them with `@`; for + instance, `@wheel` means all users in the `wheel` group. The default + is `root`. + + > **Warning** + > + > Adding a user to `trusted-users` is essentially equivalent to + > giving that user root access to the system. For example, the user + > can set `sandbox-paths` and thereby obtain read access to + > directories that are otherwise inacessible to them. + )"}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting<Strings> allowedUsers{ + this, {"*"}, "allowed-users", + R"( + A list of names of users (separated by whitespace) that are allowed + to connect to the Nix daemon. As with the `trusted-users` option, + you can specify groups by prefixing them with `@`. Also, you can + allow all users by specifying `*`. The default is `*`. + + Note that trusted users are always allowed to connect. + )"}; +}; + +AuthorizationSettings authorizationSettings; + +static GlobalConfig::Register rSettings(&authorizationSettings); + #ifndef __linux__ #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) @@ -203,8 +240,8 @@ static void daemonLoop() struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - Strings trustedUsers = settings.trustedUsers; - Strings allowedUsers = settings.allowedUsers; + Strings trustedUsers = authorizationSettings.trustedUsers; + Strings allowedUsers = authorizationSettings.allowedUsers; if (matchUser(user, group, trustedUsers)) trusted = Trusted; @@ -241,15 +278,7 @@ static void daemonLoop() // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { -#if 0 - /* Prevent users from doing something very dangerous. */ - if (geteuid() == 0 && - querySetting("build-users-group", "") == "") - throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); -#endif - store.createUser(user, peer.uid); - }); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); }, options); @@ -302,7 +331,7 @@ static void runDaemon(bool stdio) /* Auth hook is empty because in this mode we blindly trust the standard streams. Limiting access to those is explicitly not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); } } else daemonLoop(); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 16bbd8613..9d07a7a85 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,5 +1,6 @@ #include "eval.hh" #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0621d662c..3489cc132 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -106,7 +106,7 @@ void printClosureDiff( using namespace nix; -struct CmdDiffClosures : SourceExprCommand +struct CmdDiffClosures : SourceExprCommand, MixOperateOnOptions { std::string _before, _after; diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 76a134b1f..dfe75fbdf 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "attr-path.hh" #include "progress-bar.hh" +#include "editor-for.hh" #include <unistd.h> diff --git a/src/nix/eval.cc b/src/nix/eval.cc index ccee074e9..a579213fd 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -11,13 +11,13 @@ using namespace nix; -struct CmdEval : MixJSON, InstallableCommand +struct CmdEval : MixJSON, InstallableCommand, MixReadOnlyOption { bool raw = false; std::optional<std::string> apply; std::optional<Path> writeTo; - CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) + CmdEval() : InstallableCommand() { addFlag({ .longName = "raw", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 020c1b182..053a9c9e1 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "eval.hh" @@ -966,6 +967,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun struct CmdFlakeShow : FlakeCommand, MixJSON { bool showLegacy = false; + bool showAllSystems = false; CmdFlakeShow() { @@ -974,6 +976,11 @@ struct CmdFlakeShow : FlakeCommand, MixJSON .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); + addFlag({ + .longName = "all-systems", + .description = "Show the contents of outputs for all systems.", + .handler = {&showAllSystems, true} + }); } std::string description() override @@ -994,6 +1001,62 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto state = getEvalState(); auto flake = std::make_shared<LockedFlake>(lockFlake()); + auto localSystem = std::string(settings.thisSystem.get()); + + std::function<bool( + eval_cache::AttrCursor & visitor, + const std::vector<Symbol> &attrPath, + const Symbol &attr)> hasContent; + + // For frameworks it's important that structures are as lazy as possible + // to prevent infinite recursions, performance issues and errors that + // aren't related to the thing to evaluate. As a consequence, they have + // to emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not useful to the user + // so we omit them. + hasContent = [&]( + eval_cache::AttrCursor & visitor, + const std::vector<Symbol> &attrPath, + const Symbol &attr) -> bool + { + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto attrPathS = state->symbols.resolve(attrPath2); + const auto & attrName = state->symbols[attr]; + + auto visitor2 = visitor.getAttr(attrName); + + if ((attrPathS[0] == "apps" + || attrPathS[0] == "checks" + || attrPathS[0] == "devShells" + || attrPathS[0] == "legacyPackages" + || attrPathS[0] == "packages") + && (attrPathS.size() == 1 || attrPathS.size() == 2)) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + if ((attrPathS.size() == 1) + && (attrPathS[0] == "formatter" + || attrPathS[0] == "nixosConfigurations" + || attrPathS[0] == "nixosModules" + || attrPathS[0] == "overlays" + )) { + for (const auto &subAttr : visitor2->getAttrs()) { + if (hasContent(*visitor2, attrPath2, subAttr)) { + return true; + } + } + return false; + } + + // If we don't recognize it, it's probably content + return true; + }; std::function<nlohmann::json( eval_cache::AttrCursor & visitor, @@ -1020,7 +1083,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { if (!json) logger->cout("%s", headerPrefix); - auto attrs = visitor.getAttrs(); + std::vector<Symbol> attrs; + for (const auto &attr : visitor.getAttrs()) { + if (hasContent(visitor, attrPath, attr)) + attrs.push_back(attr); + } + for (const auto & [i, attr] : enumerate(attrs)) { const auto & attrName = state->symbols[attr]; bool last = i + 1 == attrs.size(); @@ -1084,10 +1152,18 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells")) ) { - if (visitor.isDerivation()) - showDerivation(); - else - throw Error("expected a derivation"); + if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } + } else { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } } else if (attrPath.size() > 0 && attrPathS[0] == "hydraJobs") { @@ -1106,6 +1182,12 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else { logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); } + } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { + if (!json) + logger->cout(fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", headerPrefix)); + else { + logger->warn(fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); + } } else { if (visitor.isDerivation()) showDerivation(); diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3c3b7bb45..5c44510ab 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -1,10 +1,13 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "finally.hh" + +#include <nlohmann/json.hpp> using namespace nix; -struct CmdPingStore : StoreCommand +struct CmdPingStore : StoreCommand, MixJSON { std::string description() override { @@ -20,10 +23,21 @@ struct CmdPingStore : StoreCommand void run(ref<Store> store) override { - notice("Store URL: %s", store->getUri()); - store->connect(); - if (auto version = store->getVersion()) - notice("Version: %s", *version); + if (!json) { + notice("Store URL: %s", store->getUri()); + store->connect(); + if (auto version = store->getVersion()) + notice("Version: %s", *version); + } else { + nlohmann::json res; + Finally printRes([&]() { + logger->cout("%s", res); + }); + res["url"] = store->getUri(); + store->connect(); + if (auto version = store->getVersion()) + res["version"] = *version; + } } }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index df9933d29..209517b21 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -68,7 +68,7 @@ std::tuple<StorePath, Hash> prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - { + .hash = { .method = ingestionMethod, .hash = *expectedHash, }, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 345505532..04ac48f00 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" @@ -202,7 +203,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narHash, }, diff --git a/src/nix/repl.cc b/src/nix/repl.cc new file mode 100644 index 000000000..679bdea77 --- /dev/null +++ b/src/nix/repl.cc @@ -0,0 +1,95 @@ +#include "eval.hh" +#include "globals.hh" +#include "command.hh" +#include "repl.hh" + +namespace nix { + +struct CmdRepl : InstallablesCommand +{ + CmdRepl() { + evalSettings.pureEval = false; + } + + void prepare() override + { + if (!settings.isExperimentalFeatureEnabled(Xp::ReplFlake) && !(file) && this->_installables.size() >= 1) { + warn("future versions of Nix will require using `--file` to load a file"); + if (this->_installables.size() > 1) + warn("more than one input file is not currently supported"); + auto filePath = this->_installables[0].data(); + file = std::optional(filePath); + _installables.front() = _installables.back(); + _installables.pop_back(); + } + installables = InstallablesCommand::load(); + } + + std::vector<std::string> files; + + Strings getDefaultFlakeAttrPaths() override + { + return {""}; + } + + bool useDefaultInstallables() override + { + return file.has_value() or expr.has_value(); + } + + bool forceImpureByDefault() override + { + return true; + } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + std::string doc() override + { + return + #include "repl.md" + ; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + auto getValues = [&]()->AbstractNixRepl::AnnotatedValues{ + auto installables = load(); + AbstractNixRepl::AnnotatedValues values; + for (auto & installable: installables){ + auto what = installable->what(); + if (file){ + auto [val, pos] = installable->toValue(*state); + auto what = installable->what(); + state->forceValue(*val, pos); + auto autoArgs = getAutoArgs(*state); + auto valPost = state->allocValue(); + state->autoCallFunction(*autoArgs, *val, *valPost); + state->forceValue(*valPost, pos); + values.push_back( {valPost, what }); + } else { + auto [val, pos] = installable->toValue(*state); + values.push_back( {val, what} ); + } + } + return values; + }; + auto repl = AbstractNixRepl::create( + searchPath, + openStore(), + state, + getValues + ); + repl->autoArgs = getAutoArgs(*repl->state); + repl->initEnv(); + repl->mainLoop(); + } +}; + +static auto rCmdRepl = registerCommand<CmdRepl>("repl"); + +} diff --git a/src/nix/search.cc b/src/nix/search.cc index d2a31607d..4fa1e7837 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -56,8 +56,8 @@ struct CmdSearch : InstallableCommand, MixJSON Strings getDefaultFlakeAttrPaths() override { return { - "packages." + settings.thisSystem.get() + ".", - "legacyPackages." + settings.thisSystem.get() + "." + "packages." + settings.thisSystem.get(), + "legacyPackages." + settings.thisSystem.get() }; } diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 29944e748..3530584f9 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -9,15 +9,44 @@ using namespace nix; struct CmdShowConfig : Command, MixJSON { + std::optional<std::string> name; + + CmdShowConfig() { + expectArgs({ + .label = {"name"}, + .optional = true, + .handler = {&name}, + }); + } + std::string description() override { - return "show the Nix configuration"; + return "show the Nix configuration or the value of a specific setting"; } Category category() override { return catUtility; } void run() override { + if (name) { + if (json) { + throw UsageError("'--json' is not supported when specifying a setting name"); + } + + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + auto setting = settings.find(*name); + + if (setting == settings.end()) { + throw Error("could not find setting '%1%'", *name); + } else { + const auto & value = setting->second.value; + logger->cout("%s", value); + } + + return; + } + if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). logger->cout("%s", globalConfig.toJSON().dump()); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 9d69c19f2..d1a516cad 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -54,57 +54,8 @@ struct CmdShowDerivation : InstallablesCommand for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - json& drvObj = jsonRoot[store->printStorePath(drvPath)]; - - auto drv = store->readDerivation(drvPath); - - { - json& outputsObj = drvObj["outputs"]; - outputsObj = json::object(); - for (auto & [_outputName, output] : drv.outputs) { - auto & outputName = _outputName; // work around clang bug - auto& outputObj = outputsObj[outputName]; - outputObj = json::object(); - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doi) { - outputObj["path"] = store->printStorePath(doi.path); - }, - [&](const DerivationOutput::CAFixed & dof) { - outputObj["path"] = store->printStorePath(dof.path(*store, drv.name, outputName)); - outputObj["hashAlgo"] = printMethodAlgo(dof.ca); - outputObj["hash"] = getContentAddressHash(dof.ca).to_string(Base16, false); - // FIXME print refs? - }, - [&](const DerivationOutput::CAFloating & dof) { - outputObj["hashAlgo"] = makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType); - }, - [&](const DerivationOutput::Deferred &) {}, - [&](const DerivationOutput::Impure & doi) { - outputObj["hashAlgo"] = makeContentAddressingPrefix(doi.method) + printHashType(doi.hashType); - outputObj["impure"] = true; - }, - }, output.raw()); - } - } - - { - auto& inputsList = drvObj["inputSrcs"]; - inputsList = json::array(); - for (auto & input : drv.inputSrcs) - inputsList.emplace_back(store->printStorePath(input)); - } - - { - auto& inputDrvsObj = drvObj["inputDrvs"]; - inputDrvsObj = json::object(); - for (auto & input : drv.inputDrvs) - inputDrvsObj[store->printStorePath(input.first)] = input.second; - } - - drvObj["system"] = drv.platform; - drvObj["builder"] = drv.builder; - drvObj["args"] = drv.args; - drvObj["env"] = drv.env; + jsonRoot[store->printStorePath(drvPath)] = + store->readDerivation(drvPath).toJSON(*store); } std::cout << jsonRoot.dump(2) << std::endl; } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 76125e5e4..a3a9dc698 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -27,7 +27,7 @@ static std::string filterPrintable(const std::string & s) return res; } -struct CmdWhyDepends : SourceExprCommand +struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions { std::string _package, _dependency; bool all = false; |