diff options
97 files changed, 1867 insertions, 1000 deletions
diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 46ced7ff7..73fb7e75e 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -10,3 +10,12 @@ git-repository-url = "https://github.com/NixOS/nix" [preprocessor.anchors] renderers = ["html"] command = "jq --from-file doc/manual/anchors.jq" + +[output.linkcheck] +# no Internet during the build (in the sandbox) +follow-web-links = false + +# mdbook-linkcheck does not understand [foo]{#bar} style links, resulting in +# excessive "Potential incomplete link" warnings. No other kind of warning was +# produced at the time of writing. +warning-policy = "ignore" diff --git a/doc/manual/local.mk b/doc/manual/local.mk index c0f69e00f..190f0258a 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -50,11 +50,16 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ - $(trace-gen) $(nix-eval) --write-to $@ --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix { toplevel = builtins.readFile $<; }' + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable + $(trace-gen) sed -i $@.tmp/*.md -e 's^@docroot@^../..^g' + @mv $@.tmp $@ $(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/generate-options.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-options.nix (builtins.fromJSON (builtins.readFile $<))' \ + | sed -e 's^@docroot@^..^g'>> $@.tmp @mv $@.tmp $@ $(d)/nix.json: $(bindir)/nix @@ -67,7 +72,9 @@ $(d)/conf-file.json: $(bindir)/nix $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp + # @docroot@: https://nixos.org/manual/nix/unstable/contributing/hacking.html#docroot-variable + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' \ + | sed -e 's^@docroot@^..^g' >> $@.tmp @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ @@ -102,6 +109,12 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli @touch $@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md - $(trace-gen) RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual + $(trace-gen) \ + set -euo pipefail; \ + RUST_LOG=warn mdbook build doc/manual -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; } + @rm -rf $(DESTDIR)$(docdir)/manual + @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual + @rm -rf $(DESTDIR)$(docdir)/manual.tmp endif diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 2d1b26558..e51958052 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -68,7 +68,7 @@ It can also execute build plans to produce new data, which are made available to A build plan itself is a series of *build tasks*, together with their build inputs. > **Important** -> A build task in Nix is called [derivation](../glossary#gloss-derivation). +> A build task in Nix is called [derivation](../glossary.md#gloss-derivation). Each build task has a special build input executed as *build instructions* in order to perform the build. The result of a build task can be input to another build task. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 5845bdc43..bb85a6b07 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -11,7 +11,7 @@ Most Nix commands interpret the following environment variables: expressions using [paths](../language/values.md#type-path) enclosed in angle brackets (i.e., `<path>`), e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](./opt-common#opt-I). + [`-I` option](./opt-common.md#opt-I). - [`NIX_IGNORE_SYMLINK_STORE`]{#env-NIX_IGNORE_SYMLINK_STORE}\ Normally, the Nix store directory (typically `/nix/store`) is not diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 83e8a2936..cd8e351bb 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`. - `--include-outputs`\ Also copy the outputs of [store derivation]s included in the closure. - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: ../glossary.md#gloss-store-derivation - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index acf29e4aa..6d0e02ca5 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -155,6 +155,12 @@ To test whether a previously-built derivation is deterministic: $ nix-build '<nixpkgs>' -A hello --check -K ``` +Use [`--read-log`](#operation---read-log) to show the stderr and stdout of a build: + +```console +$ nix-store --read-log $(nix-instantiate ./test.nix) +``` + # Operation `--serve` ## Synopsis diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index c9da1962f..aeb0d41b3 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -249,3 +249,36 @@ search/replaced in it for each new build. The installer now supports a `--tarball-url-prefix` flag which _may_ have solved this need? --> + +### Checking links in the manual + +The build checks for broken internal links. +This happens late in the process, so `nix build` is not suitable for iterating. +To build the manual incrementally, run: + +```console +make html -j $NIX_BUILD_CORES +``` + +In order to reflect changes to the [Makefile], clear all generated files before re-building: + +[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES +``` + +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment + +#### `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. @@ -96,6 +96,7 @@ buildPackages.flex (lib.getBin buildPackages.lowdown-nix) buildPackages.mdbook + buildPackages.mdbook-linkcheck buildPackages.autoconf-archive buildPackages.autoreconfHook buildPackages.pkg-config diff --git a/maintainers/README.md b/maintainers/README.md index 60768db0a..08d197c1b 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -36,17 +36,45 @@ Issues on the board progress through the following states: - No Status - Team members can add pull requests or issues to discuss or review together. - During the discussion meeting, the team triages new items. + To be considered, issues and pull requests must have a high-level description to provide the whole team with the necessary context at a glance. + + On every meeting, at least one item from each of the following categories is inspected: + + 1. [critical](https://github.com/NixOS/nix/labels/critical) + 2. [security](https://github.com/NixOS/nix/labels/security) + 3. [regression](https://github.com/NixOS/nix/labels/regression) + 4. [bug](https://github.com/NixOS/nix/issues?q=is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) + + - [oldest pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Acreated-asc) + - [most popular pull requests](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+sort%3Areactions-%2B1-desc) + - [oldest issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Acreated-asc) + - [most popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) + + Team members can also add pull requests or issues they would like the whole team to consider. + If there is disagreement on the general idea behind an issue or pull request, it is moved to _To discuss_, otherwise to _In review_. - To discuss - Pull requests and issues that are important and controversial are discussed by the team during discussion meetings. + Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. + As a general guideline, the order of items is determined as follows: + + - Prioritise pull requests over issues + + Contributors who took the time to implement concrete change proposals should not wait indefinitely. + + - Prioritise fixing bugs over documentation, improvements or new features + + The team values stability and accessibility higher than raw functionality. + + - Interleave issues and PRs + + This way issues without attempts at a solution get a chance to get addressed. + - In review Pull requests in this column are reviewed together during work meetings. diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 3ccd3c722..f19fb20bf 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->referencesPossiblyToSelf()) + for (auto & i : info->references) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 3d587dd7f..400696812 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -1,3 +1,15 @@ +function add_path --argument-names new_path + if type -q fish_add_path + # fish 3.2.0 or newer + fish_add_path --prepend --global $new_path + else + # older versions of fish + if not contains $new_path $fish_user_paths + set --global fish_user_paths $new_path $fish_user_paths + end + end +end + # Only execute this file once per shell. if test -n "$__ETC_PROFILE_NIX_SOURCED" exit @@ -31,5 +43,7 @@ else end end -fish_add_path --prepend --global "@localstatedir@/nix/profiles/default/bin" -fish_add_path --prepend --global "$HOME/.nix-profile/bin" +add_path "@localstatedir@/nix/profiles/default/bin" +add_path "$HOME/.nix-profile/bin" + +functions -e add_path diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 8d783d7c0..731498c76 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -1,3 +1,15 @@ +function add_path --argument-names new_path + if type -q fish_add_path + # fish 3.2.0 or newer + fish_add_path --prepend --global $new_path + else + # older versions of fish + if not contains $new_path $fish_user_paths + set --global fish_user_paths $new_path $fish_user_paths + end + end +end + if test -n "$HOME" && test -n "$USER" # Set up the per-user profile. @@ -32,6 +44,8 @@ if test -n "$HOME" && test -n "$USER" set --export --prepend --path MANPATH "$NIX_LINK/share/man" end - fish_add_path --prepend --global "$NIX_LINK/bin" + add_path "$NIX_LINK/bin" set --erase NIX_LINK end + +functions -e add_path diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 0e321e5e4..908127b4d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -34,8 +34,8 @@ MixEvalArgs::MixEvalArgs() .shortName = 'I', .description = R"( Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](./env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](../language/values.md#type-path) enclosed in angle + initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment + variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle brackets (i.e., `<nixpkgs>`). For instance, passing diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 79361e94e..5090ea6d2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,5 +1,6 @@ #include "globals.hh" #include "installables.hh" +#include "outputs-spec.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -358,7 +359,7 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix) } } -DerivedPath Installable::toDerivedPath() +DerivedPathWithInfo Installable::toDerivedPath() { auto buildables = toDerivedPaths(); if (buildables.size() != 1) @@ -401,18 +402,6 @@ struct InstallableStorePath : Installable ref<Store> store; DerivedPath req; - InstallableStorePath(ref<Store> store, StorePath && storePath) - : store(store), - req(storePath.isDerivation() - ? (DerivedPath) DerivedPath::Built { - .drvPath = std::move(storePath), - .outputs = {}, - } - : (DerivedPath) DerivedPath::Opaque { - .path = std::move(storePath), - }) - { } - InstallableStorePath(ref<Store> store, DerivedPath && req) : store(store), req(std::move(req)) { } @@ -422,21 +411,9 @@ struct InstallableStorePath : Installable return req.to_string(*store); } - DerivedPaths toDerivedPaths() override - { - return { req }; - } - - StorePathSet toDrvPaths(ref<Store> store) override + DerivedPathsWithInfo toDerivedPaths() override { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> StorePathSet { - return { bfd.drvPath }; - }, - [&](const DerivedPath::Opaque & bo) -> StorePathSet { - return { getDeriver(store, *this, bo.path) }; - }, - }, req.raw()); + return {{.path = req, .info = {} }}; } std::optional<StorePath> getStorePath() override @@ -452,52 +429,24 @@ struct InstallableStorePath : Installable } }; -DerivedPaths InstallableValue::toDerivedPaths() -{ - DerivedPaths res; - - std::map<StorePath, std::set<std::string>> drvsToOutputs; - RealisedPath::Set drvsToCopy; - - // Group by derivation, helps with .all in particular - for (auto & drv : toDerivations()) { - for (auto & outputName : drv.outputsToInstall) - drvsToOutputs[drv.drvPath].insert(outputName); - drvsToCopy.insert(drv.drvPath); - } - - for (auto & i : drvsToOutputs) - res.push_back(DerivedPath::Built { i.first, i.second }); - - return res; -} - -StorePathSet InstallableValue::toDrvPaths(ref<Store> store) -{ - StorePathSet res; - for (auto & drv : toDerivations()) - res.insert(drv.drvPath); - return res; -} - struct InstallableAttrPath : InstallableValue { SourceExprCommand & cmd; RootValue v; std::string attrPath; - OutputsSpec outputsSpec; + ExtendedOutputsSpec extendedOutputsSpec; InstallableAttrPath( ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath, - OutputsSpec outputsSpec) + ExtendedOutputsSpec extendedOutputsSpec) : InstallableValue(state) , cmd(cmd) , v(allocRootValue(v)) , attrPath(attrPath) - , outputsSpec(std::move(outputsSpec)) + , extendedOutputsSpec(std::move(extendedOutputsSpec)) { } std::string what() const override { return attrPath; } @@ -509,40 +458,54 @@ struct InstallableAttrPath : InstallableValue return {vRes, pos}; } - virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override; -}; + DerivedPathsWithInfo toDerivedPaths() override + { + auto v = toValue(*state).first; -std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations() -{ - auto v = toValue(*state).first; + Bindings & autoArgs = *cmd.getAutoArgs(*state); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); - DrvInfos drvInfos; - getDerivations(*state, *v, "", autoArgs, drvInfos, false); + // Backward compatibility hack: group results by drvPath. This + // helps keep .all output together. + std::map<StorePath, OutputsSpec> byDrvPath; - std::vector<DerivationInfo> res; - for (auto & drvInfo : drvInfos) { - auto drvPath = drvInfo.queryDrvPath(); - if (!drvPath) - throw Error("'%s' is not a derivation", what()); + for (auto & drvInfo : drvInfos) { + auto drvPath = drvInfo.queryDrvPath(); + if (!drvPath) + throw Error("'%s' is not a derivation", what()); - std::set<std::string> outputsToInstall; + auto newOutputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + for (auto & output : drvInfo.queryOutputs(false, true)) + outputsToInstall.insert(output.first); + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()); - if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - outputsToInstall = *outputNames; - else - for (auto & output : drvInfo.queryOutputs(false, std::get_if<DefaultOutputs>(&outputsSpec))) - outputsToInstall.insert(output.first); + auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); - res.push_back(DerivationInfo { - .drvPath = *drvPath, - .outputsToInstall = std::move(outputsToInstall) - }); - } + if (!didInsert) + iter->second = iter->second.union_(newOutputs); + } - return res; -} + DerivedPathsWithInfo res; + for (auto & [drvPath, outputs] : byDrvPath) + res.push_back({ + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = outputs, + }, + }); + + return res; + } +}; std::vector<std::string> InstallableFlake::getActualAttrPaths() { @@ -615,7 +578,7 @@ InstallableFlake::InstallableFlake( ref<EvalState> state, FlakeRef && flakeRef, std::string_view fragment, - OutputsSpec outputsSpec, + ExtendedOutputsSpec extendedOutputsSpec, Strings attrPaths, Strings prefixes, const flake::LockFlags & lockFlags) @@ -623,14 +586,14 @@ InstallableFlake::InstallableFlake( flakeRef(flakeRef), attrPaths(fragment == "" ? attrPaths : Strings{(std::string) fragment}), prefixes(fragment == "" ? Strings{} : prefixes), - outputsSpec(std::move(outputsSpec)), + extendedOutputsSpec(std::move(extendedOutputsSpec)), lockFlags(lockFlags) { if (cmd && cmd->getAutoArgs(*state)->size()) throw UsageError("'--arg' and '--argstr' are incompatible with flakes"); } -std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() +DerivedPathsWithInfo InstallableFlake::toDerivedPaths() { Activity act(*logger, lvlTalkative, actUnknown, fmt("evaluating derivation '%s'", what())); @@ -638,56 +601,84 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF auto attrPath = attr->getAttrPathStr(); - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); + if (!attr->isDerivation()) { - auto drvPath = attr->forceDerivation(); + // FIXME: use eval cache? + auto v = attr->forceValue(); - std::set<std::string> outputsToInstall; - std::optional<NixInt> priority; + if (v.type() == nPath) { + PathSet context; + auto storePath = state->copyPathToStore(context, Path(v.path)); + return {{ + .path = DerivedPath::Opaque { + .path = std::move(storePath), + } + }}; + } - if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { - if (aOutputSpecified->getBool()) { - if (auto aOutputName = attr->maybeGetAttr("outputName")) - outputsToInstall = { aOutputName->getString() }; + else if (v.type() == nString) { + PathSet context; + auto s = state->forceString(v, context, noPos, fmt("while evaluating the flake output attribute '%s'", attrPath)); + auto storePath = state->store->maybeParseStorePath(s); + if (storePath && context.count(std::string(s))) { + return {{ + .path = DerivedPath::Opaque { + .path = std::move(*storePath), + } + }}; + } else + throw Error("flake output attribute '%s' evaluates to the string '%s' which is not a store path", attrPath, s); } - } - else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { - if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) - for (auto & s : aOutputsToInstall->getListOfStrings()) - outputsToInstall.insert(s); - if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); + else + throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } - if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) { - outputsToInstall.clear(); - if (auto aOutputs = attr->maybeGetAttr(state->sOutputs)) - for (auto & s : aOutputs->getListOfStrings()) - outputsToInstall.insert(s); - } + auto drvPath = attr->forceDerivation(); - if (outputsToInstall.empty()) - outputsToInstall.insert("out"); + std::optional<NixInt> priority; - if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - outputsToInstall = *outputNames; + if (attr->maybeGetAttr(state->sOutputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aPriority = aMeta->maybeGetAttr("priority")) + priority = aPriority->getInt(); + } - auto drvInfo = DerivationInfo { - .drvPath = std::move(drvPath), - .outputsToInstall = std::move(outputsToInstall), - .priority = priority, - }; + return {{ + .path = DerivedPath::Built { + .drvPath = std::move(drvPath), + .outputs = std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { + std::set<std::string> outputsToInstall; + if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (aOutputSpecified->getBool()) { + if (auto aOutputName = attr->maybeGetAttr("outputName")) + outputsToInstall = { aOutputName->getString() }; + } + } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) + for (auto & s : aOutputsToInstall->getListOfStrings()) + outputsToInstall.insert(s); + } - return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; -} + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); -std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() -{ - std::vector<DerivationInfo> res; - res.push_back(std::get<2>(toDerivation())); - return res; + return OutputsSpec::Names { std::move(outputsToInstall) }; + }, + [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { + return e; + }, + }, extendedOutputsSpec.raw()), + }, + .info = { + .priority = priority, + .originalRef = flakeRef, + .resolvedRef = getLockedFlake()->flake.lockedRef, + .attrPath = attrPath, + .extendedOutputsSpec = extendedOutputsSpec, + } + }}; } std::pair<Value *, PosIdx> InstallableFlake::toValue(EvalState & state) @@ -802,12 +793,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( } for (auto & s : ss) { - auto [prefix, outputsSpec] = parseOutputsSpec(s); + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(s); result.push_back( std::make_shared<InstallableAttrPath>( state, *this, vFile, - prefix == "." ? "" : prefix, - outputsSpec)); + prefix == "." ? "" : std::string { prefix }, + extendedOutputsSpec)); } } else { @@ -815,24 +806,46 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( for (auto & s : ss) { std::exception_ptr ex; - auto found = s.rfind('^'); - if (found != std::string::npos) { - try { - result.push_back(std::make_shared<InstallableStorePath>( - store, - DerivedPath::Built::parse(*store, s.substr(0, found), s.substr(found + 1)))); - continue; - } catch (BadStorePath &) { - } catch (...) { - if (!ex) - ex = std::current_exception(); - } - } + auto [prefix_, extendedOutputsSpec_] = ExtendedOutputsSpec::parse(s); + // To avoid clang's pedantry + auto prefix = std::move(prefix_); + auto extendedOutputsSpec = std::move(extendedOutputsSpec_); - found = s.find('/'); + auto found = prefix.find('/'); if (found != std::string::npos) { try { - result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s))); + auto derivedPath = std::visit(overloaded { + // If the user did not use ^, we treat the output more liberally. + [&](const ExtendedOutputsSpec::Default &) -> DerivedPath { + // First, we accept a symlink chain or an actual store path. + auto storePath = store->followLinksToStorePath(prefix); + // Second, we see if the store path ends in `.drv` to decide what sort + // of derived path they want. + // + // This handling predates the `^` syntax. The `^*` in + // `/nix/store/hash-foo.drv^*` unambiguously means "do the + // `DerivedPath::Built` case", so plain `/nix/store/hash-foo.drv` could + // also unambiguously mean "do the DerivedPath::Opaque` case". + // + // Issue #7261 tracks reconsidering this `.drv` dispatching. + return storePath.isDerivation() + ? (DerivedPath) DerivedPath::Built { + .drvPath = std::move(storePath), + .outputs = OutputsSpec::All {}, + } + : (DerivedPath) DerivedPath::Opaque { + .path = std::move(storePath), + }; + }, + // If the user did use ^, we just do exactly what is written. + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + return DerivedPath::Built { + .drvPath = store->parseStorePath(prefix), + .outputs = outputSpec, + }; + }, + }, extendedOutputsSpec.raw()); + result.push_back(std::make_shared<InstallableStorePath>(store, std::move(derivedPath))); continue; } catch (BadStorePath &) { } catch (...) { @@ -842,13 +855,13 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(s, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); result.push_back(std::make_shared<InstallableFlake>( this, getEvalState(), std::move(flakeRef), fragment, - outputsSpec, + extendedOutputsSpec, getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); @@ -895,13 +908,19 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal if (mode == Realise::Nothing) settings.readOnlyMode = true; + struct Aux + { + ExtraPathInfo info; + std::shared_ptr<Installable> installable; + }; + std::vector<DerivedPath> pathsToBuild; - std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap; + std::map<DerivedPath, std::vector<Aux>> backmap; for (auto & i : installables) { for (auto b : i->toDerivedPaths()) { - pathsToBuild.push_back(b); - backmap[b].push_back(i); + pathsToBuild.push_back(b.path); + backmap[b.path].push_back({.info = b.info, .installable = i}); } } @@ -914,39 +933,18 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal printMissing(store, pathsToBuild, lvlError); for (auto & path : pathsToBuild) { - for (auto & installable : backmap[path]) { + for (auto & aux : backmap[path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store->queryRealisation(outputId); - if (!realisation) - throw MissingRealisation(outputId); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign( - output, *drvOutput->second); - } - } - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}}); + auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info}}); }, }, path.raw()); } @@ -962,16 +960,22 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Instal if (!buildResult.success()) buildResult.rethrow(); - for (auto & installable : backmap[buildResult.path]) { + for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { std::map<std::string, StorePath> outputs; for (auto & path : buildResult.builtOutputs) outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Built { bfd.drvPath, outputs }, + .info = aux.info, + .result = buildResult}}); }, [&](const DerivedPath::Opaque & bo) { - res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}}); + res.push_back({aux.installable, { + .path = BuiltPath::Opaque { bo.path }, + .info = aux.info, + .result = buildResult}}); }, }, buildResult.path.raw()); } @@ -1056,7 +1060,7 @@ StorePathSet Installable::toDerivations( [&](const DerivedPath::Built & bfd) { drvPaths.insert(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); return drvPaths; } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 02ea351d3..3d12639b0 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -2,7 +2,7 @@ #include "util.hh" #include "path.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derived-path.hh" #include "eval.hh" #include "store-api.hh" @@ -20,7 +20,7 @@ namespace eval_cache { class EvalCache; class AttrCursor; } struct App { - std::vector<StorePathWithOutputs> context; + std::vector<DerivedPath> context; Path program; // FIXME: add args, sandbox settings, metadata, ... }; @@ -52,26 +52,42 @@ enum class OperateOn { Derivation }; +struct ExtraPathInfo +{ + std::optional<NixInt> priority; + std::optional<FlakeRef> originalRef; + std::optional<FlakeRef> resolvedRef; + std::optional<std::string> attrPath; + // FIXME: merge with DerivedPath's 'outputs' field? + std::optional<ExtendedOutputsSpec> extendedOutputsSpec; +}; + +/* A derived path with any additional info that commands might + need from the derivation. */ +struct DerivedPathWithInfo +{ + DerivedPath path; + ExtraPathInfo info; +}; + struct BuiltPathWithResult { BuiltPath path; + ExtraPathInfo info; std::optional<BuildResult> result; }; +typedef std::vector<DerivedPathWithInfo> DerivedPathsWithInfo; + struct Installable { virtual ~Installable() { } virtual std::string what() const = 0; - virtual DerivedPaths toDerivedPaths() = 0; - - virtual StorePathSet toDrvPaths(ref<Store> store) - { - throw Error("'%s' cannot be converted to a derivation path", what()); - } + virtual DerivedPathsWithInfo toDerivedPaths() = 0; - DerivedPath toDerivedPath(); + DerivedPathWithInfo toDerivedPath(); UnresolvedApp toApp(EvalState & state); @@ -146,19 +162,6 @@ struct InstallableValue : Installable ref<EvalState> state; InstallableValue(ref<EvalState> state) : state(state) {} - - struct DerivationInfo - { - StorePath drvPath; - std::set<std::string> outputsToInstall; - std::optional<NixInt> priority; - }; - - virtual std::vector<DerivationInfo> toDerivations() = 0; - - DerivedPaths toDerivedPaths() override; - - StorePathSet toDrvPaths(ref<Store> store) override; }; struct InstallableFlake : InstallableValue @@ -166,7 +169,7 @@ struct InstallableFlake : InstallableValue FlakeRef flakeRef; Strings attrPaths; Strings prefixes; - OutputsSpec outputsSpec; + ExtendedOutputsSpec extendedOutputsSpec; const flake::LockFlags & lockFlags; mutable std::shared_ptr<flake::LockedFlake> _lockedFlake; @@ -175,7 +178,7 @@ struct InstallableFlake : InstallableValue ref<EvalState> state, FlakeRef && flakeRef, std::string_view fragment, - OutputsSpec outputsSpec, + ExtendedOutputsSpec extendedOutputsSpec, Strings attrPaths, Strings prefixes, const flake::LockFlags & lockFlags); @@ -186,9 +189,7 @@ struct InstallableFlake : InstallableValue Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation(); - - std::vector<DerivationInfo> toDerivations() override; + DerivedPathsWithInfo toDerivedPaths() override; std::pair<Value *, PosIdx> toValue(EvalState & state) override; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 71a7e079a..9b12f8fa2 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -641,7 +641,12 @@ bool NixRepl::processLine(std::string line) Path drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b" || command == ":bl") { - state->store->buildPaths({DerivedPath::Built{drvPath}}); + state->store->buildPaths({ + DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index afe575fee..1219b2471 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(cfg, s)); + context.push_back(NixStringContextElem::parse(cfg, s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -592,7 +592,18 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if<string_t>(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - if (!root->state.store->isValidPath(c.first)) { + const StorePath & path = std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { + return d.drvPath; + }, + [&](const NixStringContextElem::Built & b) -> const StorePath & { + return b.drvPath; + }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath & { + return o.path; + }, + }, c.raw()); + if (!root->state.store->isValidPath(path)) { valid = false; break; } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 72c2b104f..277cbb5f9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2068,27 +2068,6 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string } -/* Decode a context string ‘!<name>!<path>’ into a pair <path, - name>. */ -NixStringContextElem decodeContext(const Store & store, std::string_view s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return { - store.parseStorePath(s.substr(index + 1)), - std::string(s.substr(1, index - 1)), - }; - } else - return { - store.parseStorePath( - s.at(0) == '/' - ? s - : s.substr(1)), - "", - }; -} - - void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -2103,7 +2082,7 @@ NixStringContext Value::getContext(const Store & store) assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) - res.push_back(decodeContext(store, *p)); + res.push_back(NixStringContextElem::parse(store, *p)); return res; } @@ -2166,7 +2145,7 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet if (canonicalizePath) path = canonPath(*path); if (copyToStore) - path = copyPathToStore(context, std::move(path).toOwned()); + path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned())); return path; } @@ -2215,26 +2194,26 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value & v, PathSet } -std::string EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>(); - Path dstPath; - auto i = srcToStore.find(path); - if (i != srcToStore.end()) - dstPath = store->printStorePath(i->second); - else { - auto p = settings.readOnlyMode + auto dstPath = [&]() -> StorePath + { + auto i = srcToStore.find(path); + if (i != srcToStore.end()) return i->second; + + auto dstPath = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - dstPath = store->printStorePath(p); - allowPath(p); - srcToStore.insert_or_assign(path, std::move(p)); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); - } + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); - context.insert(dstPath); + context.insert(store->printStorePath(dstPath)); return dstPath; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9b3d160ea..46b8cbaa5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -379,7 +379,7 @@ public: bool canonicalizePath = true, std::string_view errorCtx = ""); - std::string copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(PathSet & context, const Path & path); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute @@ -551,10 +551,6 @@ struct DebugTraceStacker { std::string_view showType(ValueType type); std::string showType(const Value & v); -/* Decode a context string ‘!<name>!<path>’ into a pair <path, - name>. */ -NixStringContextElem decodeContext(const Store & store, std::string_view s); - /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index eede493f8..08adbe0c9 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -238,15 +238,15 @@ std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } -std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec( +std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec( const std::string & url, const std::optional<Path> & baseDir, bool allowMissing, bool isFlake) { - auto [prefix, outputsSpec] = parseOutputsSpec(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(prefix, baseDir, allowMissing, isFlake); - return {std::move(flakeRef), fragment, outputsSpec}; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + return {std::move(flakeRef), fragment, extendedOutputsSpec}; } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a36d852a8..c4142fc20 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "hash.hh" #include "fetchers.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include <variant> @@ -80,7 +80,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( const std::string & url, const std::optional<Path> & baseDir = {}); -std::tuple<FlakeRef, std::string, OutputsSpec> parseFlakeRefWithFragmentAndOutputsSpec( +std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec( const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false, diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 016631647..2171e769b 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,6 +6,7 @@ libexpr_DIR := $(d) libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ @@ -37,6 +38,8 @@ clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexe $(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/value/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d11b29de..f3bd0557b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -43,16 +43,32 @@ StringMap EvalState::realiseContext(const PathSet & context) std::vector<DerivedPath::Built> drvs; StringMap res; - for (auto & i : context) { - auto [ctx, outputName] = decodeContext(*store, i); - auto ctxS = store->printStorePath(ctx); - if (!store->isValidPath(ctx)) - debugThrowLastTrace(InvalidPathError(store->printStorePath(ctx))); - if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back({ctx, {outputName}}); - } else { - res.insert_or_assign(ctxS, ctxS); - } + for (auto & c_ : context) { + auto ensureValid = [&](const StorePath & p) { + if (!store->isValidPath(p)) + debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); + }; + auto c = NixStringContextElem::parse(*store, c_); + std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + drvs.push_back(DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }); + ensureValid(b.drvPath); + }, + [&](const NixStringContextElem::Opaque & o) { + auto ctxS = store->printStorePath(o.path); + res.insert_or_assign(ctxS, ctxS); + ensureValid(o.path); + }, + [&](const NixStringContextElem::DrvDeep & d) { + /* Treat same as Opaque */ + auto ctxS = store->printStorePath(d.drvPath); + res.insert_or_assign(ctxS, ctxS); + ensureValid(d.drvPath); + }, + }, c.raw()); } if (drvs.empty()) return {}; @@ -68,16 +84,12 @@ StringMap EvalState::realiseContext(const PathSet & context) store->buildPaths(buildReqs); /* Get all the output paths corresponding to the placeholders we had */ - for (auto & [drvPath, outputs] : drvs) { - const auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - auto outputPath = get(outputPaths, outputName); - if (!outputPath) - debugThrowLastTrace(Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName)); + for (auto & drv : drvs) { + auto outputs = resolveDerivedPath(*store, drv); + for (auto & [outputName, outputPath] : outputs) { res.insert_or_assign( - downstreamPlaceholder(*store, drvPath, outputName), - store->printStorePath(*outputPath) + downstreamPlaceholder(*store, drv.drvPath, outputName), + store->printStorePath(outputPath) ); } } @@ -240,6 +252,7 @@ static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { static RegisterPrimOp primop_import({ .name = "import", .args = {"path"}, + // TODO turn "normal path values" into link below .doc = R"( Load, parse and return the Nix expression in the file *path*. If *path* is a directory, the file ` default.nix ` in that directory @@ -253,7 +266,7 @@ static RegisterPrimOp primop_import({ > > Unlike some languages, `import` is a regular function in Nix. > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*) - > are [normal path values](language-values.md). + > are normal [path values](@docroot@/language/values.md#type-path). A Nix expression loaded by `import` must not contain any *free variables* (identifiers that are not defined in the Nix expression @@ -1180,38 +1193,31 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & path : context) { - - /* Paths marked with `=' denote that the path of a derivation - is explicitly passed to the builder. Since that allows the - builder to gain access to every path in the dependency - graph of the derivation (including all outputs), all paths - in the graph must be added to this derivation's list of - inputs to ensure that they are available when the builder - runs. */ - if (path.at(0) == '=') { - /* !!! This doesn't work if readOnlyMode is set. */ - StorePathSet refs; - state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); - for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (j.isDerivation()) { - Derivation jDrv = state.store->readDerivation(j); - if(jDrv.type().hasKnownOutputPaths()) - drv.inputDrvs[j] = jDrv.outputNames(); + for (auto & c_ : context) { + auto c = NixStringContextElem::parse(*state.store, c_); + std::visit(overloaded { + /* Since this allows the builder to gain access to every + path in the dependency graph of the derivation (including + all outputs), all paths in the graph must be added to + this derivation's list of inputs to ensure that they are + available when the builder runs. */ + [&](const NixStringContextElem::DrvDeep & d) { + /* !!! This doesn't work if readOnlyMode is set. */ + StorePathSet refs; + state.store->computeFSClosure(d.drvPath, refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (j.isDerivation()) + drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); } - } - } - - /* Handle derivation outputs of the form ‘!<name>!<path>’. */ - else if (path.at(0) == '!') { - auto ctx = decodeContext(*state.store, path); - drv.inputDrvs[ctx.first].insert(ctx.second); - } - - /* Otherwise it's a source file. */ - else - drv.inputSrcs.insert(state.store->parseStorePath(path)); + }, + [&](const NixStringContextElem::Built & b) { + drv.inputDrvs[b.drvPath].insert(b.output); + }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(o.path); + }, + }, c.raw()); } /* Do we have all required attributes? */ @@ -1532,8 +1538,7 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V StorePathSet refs; if (state.store->isInStore(path)) { try { - // FIXME: Are self references becoming non-self references OK? - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1876,8 +1881,7 @@ static RegisterPrimOp primop_toFile({ path. The file has suffix *name*. This file can be used as an input to derivations. One application is to write builders “inline”. For instance, the following Nix expression combines the - [Nix expression for GNU Hello](expression-syntax.md) and its - [build script](build-script.md) into one file: + Nix expression for GNU Hello and its build script into one file: ```nix { stdenv, fetchurl, perl }: @@ -1921,7 +1925,7 @@ static RegisterPrimOp primop_toFile({ ``` Note that `${configFile}` is a - [string interpolation](language/values.md#type-string), so the result of the + [string interpolation](@docroot@/language/values.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. @@ -1969,7 +1973,7 @@ static void addPath( try { auto [storePath, subPath] = state.store->toStorePath(path); // FIXME: we should scanForReferences on the path before adding it - refs = state.store->queryPathInfo(storePath)->referencesPossiblyToSelf(); + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } catch (Error &) { // FIXME: should be InvalidPathError } @@ -2817,7 +2821,7 @@ static RegisterPrimOp primop_map({ example, ```nix - map (x"foo" + x) [ "bar" "bla" "abc" ] + map (x: "foo" + x) [ "bar" "bla" "abc" ] ``` evaluates to `[ "foobar" "foobla" "fooabc" ]`. diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 9fae0b14d..0c65a6b98 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -37,8 +37,15 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + for (auto && p : context) { + auto c = NixStringContextElem::parse(*state.store, p); + if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) { + context2.emplace(state.store->printStorePath(ptr->drvPath)); + } else { + /* Can reuse original item */ + context2.emplace(std::move(p)); + } + } v.mkString(*s, context2); } @@ -74,34 +81,22 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, }; PathSet context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); - auto contextInfos = std::map<Path, ContextInfo>(); + auto contextInfos = std::map<StorePath, ContextInfo>(); for (const auto & p : context) { Path drv; std::string output; - const Path * path = &p; - if (p.at(0) == '=') { - drv = std::string(p, 1); - path = &drv; - } else if (p.at(0) == '!') { - NixStringContextElem ctx = decodeContext(*state.store, p); - drv = state.store->printStorePath(ctx.first); - output = ctx.second; - path = &drv; - } - auto isPath = drv.empty(); - auto isAllOutputs = (!drv.empty()) && output.empty(); - - auto iter = contextInfos.find(*path); - if (iter == contextInfos.end()) { - contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); - } else { - if (isPath) - iter->second.path = true; - else if (isAllOutputs) - iter->second.allOutputs = true; - else - iter->second.outputs.emplace_back(std::move(output)); - } + NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + std::visit(overloaded { + [&](NixStringContextElem::DrvDeep & d) { + contextInfos[d.drvPath].allOutputs = true; + }, + [&](NixStringContextElem::Built & b) { + contextInfos[b.drvPath].outputs.emplace_back(std::move(output)); + }, + [&](NixStringContextElem::Opaque & o) { + contextInfos[o.path].path = true; + }, + }, ctx.raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -120,7 +115,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, for (const auto & [i, output] : enumerate(info.second.outputs)) (outputsVal.listElems()[i] = state.allocValue())->mkString(output); } - attrs.alloc(info.first).mkAttrs(infoAttrs); + attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); } v.mkAttrs(attrs); diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index b95980cab..e483575a4 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -6,7 +6,9 @@ libexpr-tests_DIR := $(d) libexpr-tests_INSTALL_DIR := -libexpr-tests_SOURCES := $(wildcard $(d)/*.cc) +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc new file mode 100644 index 000000000..d5c9d3bce --- /dev/null +++ b/src/libexpr/tests/value/context.cc @@ -0,0 +1,72 @@ +#include "value/context.hh" + +#include "libexprtests.hh" + +namespace nix { + +// Testing of trivial expressions +struct NixStringContextElemTest : public LibExprTest { + const Store & store() const { + return *LibExprTest::store; + } +}; + +TEST_F(NixStringContextElemTest, empty_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), ""), + BadNixStringContextElem); +} + +TEST_F(NixStringContextElemTest, single_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "!"), + BadNixStringContextElem); +} + +TEST_F(NixStringContextElemTest, double_bang_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "!!/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, eq_slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "=/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, slash_invalid) { + EXPECT_THROW( + NixStringContextElem::parse(store(), "/"), + BadStorePath); +} + +TEST_F(NixStringContextElemTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(store(), opaque); + auto * p = std::get_if<NixStringContextElem::Opaque>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store().parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(store()), opaque); +} + +TEST_F(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(store(), drvDeep); + auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); + ASSERT_EQ(elem.to_string(store()), drvDeep); +} + +TEST_F(NixStringContextElemTest, built) { + std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(store(), built); + auto * p = std::get_if<NixStringContextElem::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); + ASSERT_EQ(elem.to_string(store()), built); +} + +} diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5dc453b2e..c35c876e3 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,6 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" #include "util.hh" +#include "store-api.hh" #include <cstdlib> #include <iomanip> @@ -35,7 +36,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) - out = state.copyPathToStore(context, v.path); + out = state.store->printStorePath(state.copyPathToStore(context, v.path)); else out = v.path; break; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index f57597cff..7d3f6d700 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include <cassert> #include "symbol-table.hh" +#include "value/context.hh" #if HAVE_BOEHMGC #include <gc/gc_allocator.h> @@ -67,8 +68,6 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -typedef std::pair<StorePath, std::string> NixStringContextElem; -typedef std::vector<NixStringContextElem> NixStringContext; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc new file mode 100644 index 000000000..61d9c53df --- /dev/null +++ b/src/libexpr/value/context.cc @@ -0,0 +1,67 @@ +#include "value/context.hh" +#include "store-api.hh" + +#include <optional> + +namespace nix { + +NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +{ + std::string_view s = s0; + + if (s.size() == 0) { + throw BadNixStringContextElem(s0, + "String context element should never be an empty string"); + } + switch (s.at(0)) { + case '!': { + s = s.substr(1); // advance string to parse after first ! + size_t index = s.find("!"); + // This makes index + 1 safe. Index can be the length (one after index + // of last character), so given any valid character index --- a + // successful find --- we can add one. + if (index == std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element beginning with '!' should have a second '!'"); + } + return NixStringContextElem::Built { + .drvPath = store.parseStorePath(s.substr(index + 1)), + .output = std::string(s.substr(0, index)), + }; + } + case '=': { + return NixStringContextElem::DrvDeep { + .drvPath = store.parseStorePath(s.substr(1)), + }; + } + default: { + return NixStringContextElem::Opaque { + .path = store.parseStorePath(s), + }; + } + } +} + +std::string NixStringContextElem::to_string(const Store & store) const { + return std::visit(overloaded { + [&](const NixStringContextElem::Built & b) { + std::string res; + res += '!'; + res += b.output; + res += '!'; + res += store.printStorePath(b.drvPath); + return res; + }, + [&](const NixStringContextElem::DrvDeep & d) { + std::string res; + res += '='; + res += store.printStorePath(d.drvPath); + return res; + }, + [&](const NixStringContextElem::Opaque & o) { + return store.printStorePath(o.path); + }, + }, raw()); +} + +} diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh new file mode 100644 index 000000000..d8008c436 --- /dev/null +++ b/src/libexpr/value/context.hh @@ -0,0 +1,90 @@ +#pragma once + +#include "util.hh" +#include "path.hh" + +#include <optional> + +#include <nlohmann/json_fwd.hpp> + +namespace nix { + +class BadNixStringContextElem : public Error +{ +public: + std::string_view raw; + + template<typename... Args> + BadNixStringContextElem(std::string_view raw_, const Args & ... args) + : Error("") + { + raw = raw_; + auto hf = hintfmt(args...); + err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw); + } +}; + +class Store; + +/* Plain opaque path to some store object. + + Encoded as just the path: ‘<path>’. +*/ +struct NixStringContextElem_Opaque { + StorePath path; +}; + +/* Path to a derivation and its entire build closure. + + The path doesn't just refer to derivation itself and its closure, but + also all outputs of all derivations in that closure (including the + root derivation). + + Encoded in the form ‘=<drvPath>’. +*/ +struct NixStringContextElem_DrvDeep { + StorePath drvPath; +}; + +/* Derivation output. + + Encoded in the form ‘!<output>!<drvPath>’. +*/ +struct NixStringContextElem_Built { + StorePath drvPath; + std::string output; +}; + +using _NixStringContextElem_Raw = std::variant< + NixStringContextElem_Opaque, + NixStringContextElem_DrvDeep, + NixStringContextElem_Built +>; + +struct NixStringContextElem : _NixStringContextElem_Raw { + using Raw = _NixStringContextElem_Raw; + using Raw::Raw; + + using Opaque = NixStringContextElem_Opaque; + using DrvDeep = NixStringContextElem_DrvDeep; + using Built = NixStringContextElem_Built; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + inline Raw & raw() { + return static_cast<Raw &>(*this); + } + + /* Decode a context string, one of: + - ‘<path>’ + - ‘=<path>’ + - ‘!<name>!<path>’ + */ + static NixStringContextElem parse(const Store & store, std::string_view s); + std::string to_string(const Store & store) const; +}; + +typedef std::vector<NixStringContextElem> NixStringContext; + +} diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3c37fd627..1715374a6 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -39,7 +39,6 @@ void printVersion(const std::string & programName); void printGCWarning(); class Store; -struct StorePathWithOutputs; void printMissing( ref<Store> store, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index aa5aafdbf..ac41add2c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -180,11 +180,11 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( duration); /* Verify that all references are valid. This may do some .narinfo - reads, but typically they'll already be cached. Note that - self-references are always valid. */ - for (auto & ref : info.references.others) + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) try { - queryPathInfo(ref); + if (ref != info.path) + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); @@ -530,22 +530,9 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe writeNarInfo(narInfo); } -std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) +std::optional<std::string> BinaryCacheStore::getBuildLogExact(const StorePath & path) { - auto drvPath = path; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return std::nullopt; - drvPath = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - - auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(path))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 8c82e2387..abd92a83c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -129,7 +129,7 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::optional<std::string> getBuildLog(const StorePath & path) override; + std::optional<std::string> getBuildLogExact(const StorePath & path) override; void addBuildLog(const StorePath & drvPath, std::string_view log) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e86b5269..2021d0023 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -63,7 +63,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) @@ -82,7 +82,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) @@ -142,18 +142,12 @@ void DerivationGoal::work() (this->*state)(); } -void DerivationGoal::addWantedOutputs(const StringSet & outputs) +void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { - /* If we already want all outputs, there is nothing to do. */ - if (wantedOutputs.empty()) return; - - if (outputs.empty()) { - wantedOutputs.clear(); + auto newWanted = wantedOutputs.union_(outputs); + if (!newWanted.isSubsetOf(wantedOutputs)) needRestart = true; - } else - for (auto & i : outputs) - if (wantedOutputs.insert(i).second) - needRestart = true; + wantedOutputs = newWanted; } @@ -390,7 +384,7 @@ void DerivationGoal::repairClosure() auto outputs = queryDerivationOutputMap(); StorePathSet outputClosure; for (auto & i : outputs) { - if (!wantOutput(i.first, wantedOutputs)) continue; + if (!wantedOutputs.contains(i.first)) continue; worker.store.computeFSClosure(i.second, outputClosure); } @@ -422,7 +416,7 @@ void DerivationGoal::repairClosure() if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); + addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); } if (waitees.empty()) { @@ -991,10 +985,15 @@ void DerivationGoal::resolvedFinished() StorePathSet outputPaths; - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); + // `wantedOutputs` might merely indicate “all the outputs” + auto realWantedOutputs = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return resolvedDrv.outputNames(); + }, + [&](const OutputsSpec::Names & names) { + return static_cast<std::set<std::string>>(names); + }, + }, wantedOutputs.raw()); for (auto & wantedOutput : realWantedOutputs) { auto initialOutput = get(initialOutputs, wantedOutput); @@ -1322,7 +1321,14 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() if (!drv->type().isPure()) return { false, {} }; bool checkHash = buildMode == bmRepair; - auto wantedOutputsLeft = wantedOutputs; + auto wantedOutputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast<StringSet>(names); + }, + }, wantedOutputs.raw()); DrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { @@ -1331,7 +1337,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() // this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) continue; auto & info = *initialOutput; - info.wanted = wantOutput(i.first, wantedOutputs); + info.wanted = wantedOutputs.contains(i.first); if (info.wanted) wantedOutputsLeft.erase(i.first); if (i.second) { @@ -1369,7 +1375,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } - // If we requested all the outputs via the empty set, we are always fine. + // If we requested all the outputs, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. if (!wantedOutputsLeft.empty()) diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index d33e04cbc..707e38b4b 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,6 +2,7 @@ #include "parsed-derivations.hh" #include "lock.hh" +#include "outputs-spec.hh" #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" @@ -55,7 +56,7 @@ struct DerivationGoal : public Goal /* The specific outputs that we need to build. Empty means all of them. */ - StringSet wantedOutputs; + OutputsSpec wantedOutputs; /* Mapping from input derivations + output names to actual store paths. This is filled in by waiteeDone() as each dependency @@ -128,10 +129,10 @@ struct DerivationGoal : public Goal std::string machineName; DerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, Worker & worker, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); virtual ~DerivationGoal(); @@ -142,7 +143,7 @@ struct DerivationGoal : public Goal void work() override; /* Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const StringSet & outputs); + void addWantedOutputs(const OutputsSpec & outputs); /* The states. */ void getDerivation(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e1b80165e..2925fe3ca 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -80,7 +80,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { Worker worker(*this, *this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); try { worker.run(Goals{goal}); @@ -89,7 +89,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat return BuildResult { .status = BuildResult::MiscFailure, .errorMsg = e.msg(), - .path = DerivedPath::Built { .drvPath = drvPath }, + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, }; }; } @@ -130,7 +133,8 @@ void LocalStore::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); + // FIXME: Should just build the specific output we need. + goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); worker.run(goals); } else throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c7b3d9325..88cb33d0f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1459,7 +1459,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo unknown, downloadSize, narSize); } - virtual std::optional<std::string> getBuildLog(const StorePath & path) override + virtual std::optional<std::string> getBuildLogExact(const StorePath & path) override { return std::nullopt; } virtual void addBuildLog(const StorePath & path, std::string_view log) override @@ -2529,7 +2529,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - newInfo0.references = rewriteRefs(); + auto refs = rewriteRefs(); + newInfo0.references = std::move(refs.others); + if (refs.self) + newInfo0.references.insert(newInfo0.path); return newInfo0; }, @@ -2738,7 +2741,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - if (wantOutput(outputName, wantedOutputs)) + if (wantedOutputs.contains(outputName)) builtOutputs.emplace(thisRealisation.id, thisRealisation); } @@ -2786,12 +2789,12 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.referencesPossiblyToSelf()) + for (auto & ref : i->second.references) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) pathsLeft.push(ref); } } @@ -2831,7 +2834,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo auto used = recursive ? getClosure(info.path).first - : info.referencesPossiblyToSelf(); + : info.references; if (recursive && checks.ignoreSelfRefs) used.erase(info.path); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 36b0ea7a7..994cb4ac2 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -165,8 +165,9 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info->references.others) - addWaitee(worker.makePathSubstitutionGoal(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -187,8 +188,9 @@ void PathSubstitutionGoal::referencesValid() return; } - for (auto & i : info->references.others) - assert(worker.store.isValidPath(i)); + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b192fbc77..b94fb8416 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -42,7 +42,7 @@ Worker::~Worker() std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( const StorePath & drvPath, - const StringSet & wantedOutputs, + const OutputsSpec & wantedOutputs, std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) { std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath]; @@ -59,7 +59,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode) + const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) @@ -70,7 +70,7 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode) + const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a1e036a96..6d68d3cf1 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -140,15 +140,15 @@ public: /* derivation goal */ private: std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( - const StorePath & drvPath, const StringSet & wantedOutputs, + const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); public: std::shared_ptr<DerivationGoal> makeDerivationGoal( const StorePath & drvPath, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, - const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 05b289ee7..5452758b3 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -208,6 +208,16 @@ Hash getContentAddressHash(const ContentAddress & ca) }, ca); } +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c07fe0a88..a251aeb01 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -4,7 +4,6 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" -#include "reference-set.hh" namespace nix { @@ -102,7 +101,15 @@ Hash getContentAddressHash(const ContentAddress & ca); * References set */ -typedef References<StorePath> StoreReferences; +struct StoreReferences { + StorePathSet others; + bool self = false; + + bool empty() const; + size_t size() const; + + GENERATE_CMP(StoreReferences, me->self, me->others); +}; /* * Full content address diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 275c2b635..0694f5b90 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -336,7 +336,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) + for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -762,7 +762,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references.possiblyToSelf(path)); + worker_proto::write(*store, to, i->second.references); to << i->second.downloadSize << i->second.narSize; } @@ -785,7 +785,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references.possiblyToSelf(i.first)); + worker_proto::write(*store, to, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; @@ -867,7 +867,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom<StorePathSet> {})); + info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c2c0520f5..2a1fbe0f6 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -686,12 +686,6 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & } -bool wantOutput(const std::string & output, const std::set<std::string> & wanted) -{ - return wanted.empty() || wanted.find(output) != wanted.end(); -} - - static DerivationOutput readDerivationOutput(Source & in, const Store & store) { const auto pathS = readString(in); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index cd6117a59..c57a6f808 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -294,8 +294,6 @@ typedef std::map<StorePath, DrvHash> DrvHashes; // FIXME: global, though at least thread-safe. extern Sync<DrvHashes> drvHashes; -bool wantOutput(const std::string & output, const std::set<std::string> & wanted); - struct Source; struct Sink; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 3fa5ae4f7..e0d86a42f 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -19,11 +19,11 @@ nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const { res["drvPath"] = store->printStorePath(drvPath); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); - for (const auto & output : outputs) { - auto knownOutput = get(knownOutputs, output); - if (knownOutput && *knownOutput) - res["outputs"][output] = store->printStorePath(**knownOutput); + const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + for (const auto & [output, outputPathOpt] : outputMap) { + if (!outputs.contains(output)) continue; + if (outputPathOpt) + res["outputs"][output] = store->printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } @@ -63,7 +63,7 @@ std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) + "!" - + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); + + outputs.to_string(); } std::string DerivedPath::to_string(const Store & store) const @@ -81,15 +81,10 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) { - auto drvPath = store.parseStorePath(drvS); - std::set<std::string> outputs; - if (outputsS != "*") { - outputs = tokenizeString<std::set<std::string>>(outputsS, ","); - if (outputs.empty()) - throw Error( - "Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS); - } - return {drvPath, outputs}; + return { + .drvPath = store.parseStorePath(drvS), + .outputs = OutputsSpec::parse(outputsS), + }; } DerivedPath DerivedPath::parse(const Store & store, std::string_view s) diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b6063c300..5f2dfa4f1 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,6 +3,7 @@ #include "util.hh" #include "path.hh" #include "realisation.hh" +#include "outputs-spec.hh" #include <optional> #include <variant> @@ -45,7 +46,7 @@ struct DerivedPathOpaque { */ struct DerivedPathBuilt { StorePath drvPath; - std::set<std::string> outputs; + OutputsSpec outputs; std::string to_string(const Store & store) const; static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4adf51573..9875da909 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); + worker_proto::write(*this, teeSink, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.setReferencesPossiblyToSelf(std::move(references)); + info.references = references; info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f026c8808..7111def92 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -676,7 +676,7 @@ public: - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-caches"}}; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a3e3d2217..2c9dd2680 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -137,7 +137,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); + info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize @@ -200,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -278,7 +278,12 @@ public: conn->to.flush(); - BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; + BuildResult status { + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index c5ae7536f..b224fc3e9 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -87,20 +87,8 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) const std::string LocalFSStore::drvsLogDir = "drvs"; -std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_) +std::optional<std::string> LocalFSStore::getBuildLogExact(const StorePath & path) { - auto path = path_; - - if (!path.isDerivation()) { - try { - auto info = queryPathInfo(path); - if (!info->deriver) return std::nullopt; - path = *info->deriver; - } catch (InvalidPath &) { - return std::nullopt; - } - } - auto baseName = path.to_string(); for (int j = 0; j < 2; j++) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index e6fb3201a..947707341 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -50,7 +50,7 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::optional<std::string> getBuildLog(const StorePath & path) override; + std::optional<std::string> getBuildLogExact(const StorePath & path) override; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 23cf2e3ef..654aee973 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -938,8 +938,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->insertReferencePossiblyToSelf( - parseStorePath(useQueryReferences.getStr(0))); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1206,7 +1205,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.referencesPossiblyToSelf()) + for (auto & j : i.references) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1227,7 +1226,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSort(paths, {[&](const StorePath & path) { auto i = infos.find(path); - return i == infos.end() ? StorePathSet() : i->second.references.others; + return i == infos.end() ? StorePathSet() : i->second.references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( @@ -1525,8 +1524,7 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); - // No self reference allowed with text-hashing - info.references.others = references; + info.references = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } diff --git a/src/libstore/log-store.cc b/src/libstore/log-store.cc new file mode 100644 index 000000000..8a26832ab --- /dev/null +++ b/src/libstore/log-store.cc @@ -0,0 +1,12 @@ +#include "log-store.hh" + +namespace nix { + +std::optional<std::string> LogStore::getBuildLog(const StorePath & path) { + auto maybePath = getBuildDerivationPath(path); + if (!maybePath) + return std::nullopt; + return getBuildLogExact(maybePath.value()); +} + +} diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index ff1b92e17..e4d95bab6 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -11,7 +11,9 @@ struct LogStore : public virtual Store /* Return the build log of the specified store path, if available, or null otherwise. */ - virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0; + std::optional<std::string> getBuildLog(const StorePath & path); + + virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0; virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 5d7945eb1..09f615439 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -28,8 +28,9 @@ std::map<StorePath, StorePath> makeContentAddressed( StringMap rewrites; StoreReferences refs; - refs.self = oldInfo->references.self; - for (auto & ref : oldInfo->references.others) { + for (auto & ref : oldInfo->references) { + if (ref == path) + refs.self = true; auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9a38a0713..dace653d6 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,8 +40,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future<ref<const ValidPathInfo>> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references.others) - res.insert(ref); + for (auto & ref : info->references) + if (ref != path) + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) @@ -196,7 +197,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, knownOutputPaths = false; break; } - if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt)) + if (bfd.outputs.contains(outputName) && !isValidPath(*pathOpt)) invalid.insert(*pathOpt); } if (knownOutputPaths && invalid.empty()) return; @@ -235,7 +236,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, state->narSize += info->second.narSize; } - for (auto & ref : info->second.references.others) + for (auto & ref : info->second.references) pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref })); }, }, req.raw()); @@ -253,7 +254,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) return topoSort(paths, {[&](const StorePath & path) { try { - return queryPathInfo(path)->references.others; + return queryPathInfo(path)->references; } catch (InvalidPath &) { return StorePathSet(); } @@ -309,7 +310,50 @@ std::map<DrvOutput, StorePath> drvOutputReferences( auto info = store.queryPathInfo(outputPath); - return drvOutputReferences(Realisation::closure(store, inputRealisations), info->referencesPossiblyToSelf()); + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); +} + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + OutputPathMap outputs; + auto drv = evalStore.readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(store, drv); + auto drvOutputs = drv.outputsAndOptPaths(store); + auto outputNames = std::visit(overloaded { + [&](const OutputsSpec::All &) { + StringSet names; + for (auto & [outputName, _] : drv.outputs) + names.insert(outputName); + return names; + }, + [&](const OutputsSpec::Names & names) { + return static_cast<std::set<std::string>>(names); + }, + }, bfd.outputs); + for (auto & output : outputNames) { + auto outputHash = get(outputHashes, output); + if (!outputHash) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store.printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + DrvOutput outputId { *outputHash, output }; + auto realisation = store.queryRealisation(outputId); + if (!realisation) + throw MissingRealisation(outputId); + outputs.insert_or_assign(output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + auto drvOutput = get(drvOutputs, output); + assert(drvOutput); + assert(drvOutput->second); + outputs.insert_or_assign(output, *drvOutput->second); + } + } + return outputs; } } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8d4a21daf..3e0689534 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -248,7 +248,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->insertReferencePossiblyToSelf(StorePath(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index f54e8f1fc..071d8355e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - insertReferencePossiblyToSelf(StorePath(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc new file mode 100644 index 000000000..d0f39a854 --- /dev/null +++ b/src/libstore/outputs-spec.cc @@ -0,0 +1,189 @@ +#include "util.hh" +#include "outputs-spec.hh" +#include "nlohmann/json.hpp" + +#include <regex> + +namespace nix { + +bool OutputsSpec::contains(const std::string & outputName) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & outputNames) { + return outputNames.count(outputName) > 0; + }, + }, raw()); +} + + +std::optional<OutputsSpec> OutputsSpec::parseOpt(std::string_view s) +{ + static std::regex regex(R"((\*)|([a-z]+(,[a-z]+)*))"); + + std::smatch match; + std::string s2 { s }; // until some improves std::regex + if (!std::regex_match(s2, match, regex)) + return std::nullopt; + + if (match[1].matched) + return { OutputsSpec::All {} }; + + if (match[2].matched) + return OutputsSpec::Names { tokenizeString<StringSet>(match[2].str(), ",") }; + + assert(false); +} + + +OutputsSpec OutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("Invalid outputs specifier: '%s'", s); + return *spec; +} + + +std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsSpec::parseOpt(std::string_view s) +{ + auto found = s.rfind('^'); + + if (found == std::string::npos) + return std::pair { s, ExtendedOutputsSpec::Default {} }; + + auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); + if (!specOpt) + return std::nullopt; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; +} + + +std::pair<std::string_view, ExtendedOutputsSpec> ExtendedOutputsSpec::parse(std::string_view s) +{ + std::optional spec = parseOpt(s); + if (!spec) + throw Error("Invalid extended outputs specifier: '%s'", s); + return *spec; +} + + +std::string OutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> std::string { + return "*"; + }, + [&](const OutputsSpec::Names & outputNames) -> std::string { + return concatStringsSep(",", outputNames); + }, + }, raw()); +} + + +std::string ExtendedOutputsSpec::to_string() const +{ + return std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) -> std::string { + return ""; + }, + [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string { + return "^" + outputSpec.to_string(); + }, + }, raw()); +} + + +OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All { }; + }, + [&](const OutputsSpec::Names & theseNames) -> OutputsSpec { + return std::visit(overloaded { + [&](const OutputsSpec::All &) -> OutputsSpec { + return OutputsSpec::All {}; + }, + [&](const OutputsSpec::Names & thoseNames) -> OutputsSpec { + OutputsSpec::Names ret = theseNames; + ret.insert(thoseNames.begin(), thoseNames.end()); + return ret; + }, + }, that.raw()); + }, + }, raw()); +} + + +bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const +{ + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return true; + }, + [&](const OutputsSpec::Names & thoseNames) { + return std::visit(overloaded { + [&](const OutputsSpec::All &) { + return false; + }, + [&](const OutputsSpec::Names & theseNames) { + bool ret = true; + for (auto & o : theseNames) + if (thoseNames.count(o) == 0) + ret = false; + return ret; + }, + }, raw()); + }, + }, that.raw()); +} + +} + +namespace nlohmann { + +using namespace nix; + +OutputsSpec adl_serializer<OutputsSpec>::from_json(const json & json) { + auto names = json.get<StringSet>(); + if (names == StringSet({"*"})) + return OutputsSpec::All {}; + else + return OutputsSpec::Names { std::move(names) }; +} + +void adl_serializer<OutputsSpec>::to_json(json & json, OutputsSpec t) { + std::visit(overloaded { + [&](const OutputsSpec::All &) { + json = std::vector<std::string>({"*"}); + }, + [&](const OutputsSpec::Names & names) { + json = names; + }, + }, t); +} + + +ExtendedOutputsSpec adl_serializer<ExtendedOutputsSpec>::from_json(const json & json) { + if (json.is_null()) + return ExtendedOutputsSpec::Default {}; + else { + return ExtendedOutputsSpec::Explicit { json.get<OutputsSpec>() }; + } +} + +void adl_serializer<ExtendedOutputsSpec>::to_json(json & json, ExtendedOutputsSpec t) { + std::visit(overloaded { + [&](const ExtendedOutputsSpec::Default &) { + json = nullptr; + }, + [&](const ExtendedOutputsSpec::Explicit & e) { + adl_serializer<OutputsSpec>::to_json(json, e); + }, + }, t); +} + +} diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh new file mode 100644 index 000000000..46bc35ebc --- /dev/null +++ b/src/libstore/outputs-spec.hh @@ -0,0 +1,95 @@ +#pragma once + +#include <cassert> +#include <optional> +#include <set> +#include <variant> + +#include "json-impls.hh" + +namespace nix { + +struct OutputNames : std::set<std::string> { + using std::set<std::string>::set; + + /* These need to be "inherited manually" */ + + OutputNames(const std::set<std::string> & s) + : std::set<std::string>(s) + { assert(!empty()); } + + OutputNames(std::set<std::string> && s) + : std::set<std::string>(s) + { assert(!empty()); } + + /* This set should always be non-empty, so we delete this + constructor in order make creating empty ones by mistake harder. + */ + OutputNames() = delete; +}; + +struct AllOutputs : std::monostate { }; + +typedef std::variant<AllOutputs, OutputNames> _OutputsSpecRaw; + +struct OutputsSpec : _OutputsSpecRaw { + using Raw = _OutputsSpecRaw; + using Raw::Raw; + + /* Force choosing a variant */ + OutputsSpec() = delete; + + using Names = OutputNames; + using All = AllOutputs; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + inline Raw & raw() { + return static_cast<Raw &>(*this); + } + + bool contains(const std::string & output) const; + + /* Create a new OutputsSpec which is the union of this and that. */ + OutputsSpec union_(const OutputsSpec & that) const; + + /* Whether this OutputsSpec is a subset of that. */ + bool isSubsetOf(const OutputsSpec & outputs) const; + + /* Parse a string of the form 'output1,...outputN' or + '*', returning the outputs spec. */ + static OutputsSpec parse(std::string_view s); + static std::optional<OutputsSpec> parseOpt(std::string_view s); + + std::string to_string() const; +}; + +struct DefaultOutputs : std::monostate { }; + +typedef std::variant<DefaultOutputs, OutputsSpec> _ExtendedOutputsSpecRaw; + +struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { + using Raw = _ExtendedOutputsSpecRaw; + using Raw::Raw; + + using Default = DefaultOutputs; + using Explicit = OutputsSpec; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + /* Parse a string of the form 'prefix^output1,...outputN' or + 'prefix^*', returning the prefix and the extended outputs spec. */ + static std::pair<std::string_view, ExtendedOutputsSpec> parse(std::string_view s); + static std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> parseOpt(std::string_view s); + + std::string to_string() const; +}; + +} + +JSON_IMPL(OutputsSpec) +JSON_IMPL(ExtendedOutputsSpec) diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 003685604..93f91e702 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,6 +3,125 @@ namespace nix { +std::string ValidPathInfo::fingerprint(const Store & store) const +{ + if (narSize == 0) + throw Error("cannot calculate fingerprint of path '%s' because its size is not known", + store.printStorePath(path)); + return + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(Base32, true) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", store.printStorePathSet(references)); +} + + +void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint(store))); +} + +std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + th, + .references = references, + }; + }, + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return FixedOutputInfo { + foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, *ca), + }; +} + +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + auto fullCaOpt = fullStorePathDescriptorOpt(); + + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); + + bool res = caPath == path; + + if (!res) + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + + return res; +} + + +size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const +{ + if (isContentAddressed(store)) return maxSigs; + + size_t good = 0; + for (auto & sig : sigs) + if (checkSignature(store, publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(store), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(std::string(r.to_string())); + return refs; +} + + +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); + }, + [this](FixedOutputInfo && foi) { + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); + this->ca = std::move((FixedOutputHash &&) foi); + }, + }, std::move(info.info)); +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -14,7 +133,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom<StorePathSet> {})); + info.references = worker_proto::read(store, source, Phantom<StorePathSet> {}); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -24,6 +143,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned return info; } + void ValidPathInfo::write( Sink & sink, const Store & store, @@ -34,7 +154,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, referencesPossiblyToSelf()); + worker_proto::write(store, sink, references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 9254835b7..476df79c2 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -17,19 +17,20 @@ class Store; struct SubstitutablePathInfo { std::optional<StorePath> deriver; - StoreReferences references; + StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; + struct ValidPathInfo { StorePath path; std::optional<StorePath> deriver; Hash narHash; - StoreReferences references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -81,12 +82,6 @@ struct ValidPathInfo /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf() const; - void insertReferencePossiblyToSelf(StorePath && ref); - void setReferencesPossiblyToSelf(StorePathSet && refs); - static const size_t maxSigs = std::numeric_limits<size_t>::max(); /* Return the number of signatures on this .narinfo that were diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index d6d67ea05..869b490ad 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,6 +1,5 @@ #include "path-with-outputs.hh" #include "store-api.hh" -#include "nlohmann/json.hpp" #include <regex> @@ -16,10 +15,14 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { - if (!outputs.empty() || path.isDerivation()) - return DerivedPath::Built { path, outputs }; - else + if (!outputs.empty()) { + return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + } else if (path.isDerivation()) { + assert(outputs.empty()); + return DerivedPath::Built { path, OutputsSpec::All { } }; + } else { return DerivedPath::Opaque { path }; + } } @@ -42,7 +45,18 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv return StorePathWithOutputs { bo.path }; }, [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> { - return StorePathWithOutputs { bfd.drvPath, bfd.outputs }; + return StorePathWithOutputs { + .path = bfd.drvPath, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast<StringSet>(outputs); + }, + }, bfd.outputs.raw()), + }; }, }, p.raw()); } @@ -53,8 +67,8 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s) size_t n = s.find("!"); return n == s.npos ? std::make_pair(s, std::set<std::string>()) - : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ",")); + : std::make_pair(s.substr(0, n), + tokenizeString<std::set<std::string>>(s.substr(n + 1), ",")); } @@ -71,57 +85,4 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } -std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s) -{ - static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); - - std::smatch match; - if (!std::regex_match(s, match, regex)) - return {s, DefaultOutputs()}; - - if (match[3].matched) - return {match[1], AllOutputs()}; - - return {match[1], tokenizeString<OutputNames>(match[4].str(), ",")}; -} - -std::string printOutputsSpec(const OutputsSpec & outputsSpec) -{ - if (std::get_if<DefaultOutputs>(&outputsSpec)) - return ""; - - if (std::get_if<AllOutputs>(&outputsSpec)) - return "^*"; - - if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - return "^" + concatStringsSep(",", *outputNames); - - assert(false); -} - -void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec) -{ - if (std::get_if<DefaultOutputs>(&outputsSpec)) - json = nullptr; - - else if (std::get_if<AllOutputs>(&outputsSpec)) - json = std::vector<std::string>({"*"}); - - else if (auto outputNames = std::get_if<OutputNames>(&outputsSpec)) - json = *outputNames; -} - -void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec) -{ - if (json.is_null()) - outputsSpec = DefaultOutputs(); - else { - auto names = json.get<OutputNames>(); - if (names == OutputNames({"*"})) - outputsSpec = AllOutputs(); - else - outputsSpec = names; - } -} - } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 0cb5eb223..5d25656a5 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,13 +1,17 @@ #pragma once -#include <variant> - #include "path.hh" #include "derived-path.hh" -#include "nlohmann/json_fwd.hpp" namespace nix { +/* This is a deprecated old type just for use by the old CLI, and older + versions of the RPC protocols. In new code don't use it; you want + `DerivedPath` instead. + + `DerivedPath` is better because it handles more cases, and does so more + explicitly without devious punning tricks. +*/ struct StorePathWithOutputs { StorePath path; @@ -33,25 +37,4 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); -typedef std::set<std::string> OutputNames; - -struct AllOutputs { - bool operator < (const AllOutputs & _) const { return false; } -}; - -struct DefaultOutputs { - bool operator < (const DefaultOutputs & _) const { return false; } -}; - -typedef std::variant<DefaultOutputs, AllOutputs, OutputNames> OutputsSpec; - -/* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the outputs spec. */ -std::pair<std::string, OutputsSpec> parseOutputsSpec(const std::string & s); - -std::string printOutputsSpec(const OutputsSpec & outputsSpec); - -void to_json(nlohmann::json &, const OutputsSpec &); -void from_json(const nlohmann::json &, OutputsSpec &); - } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 6771f0ba5..7f13c11e9 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -65,7 +65,6 @@ public: typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -typedef std::map<std::string, StorePath> OutputPathMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bfe85d83b..5f1bb73ac 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); + info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,12 +421,11 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { - auto path = parseStorePath(readString(conn->from)); - SubstitutablePathInfo & info { infos[path] }; + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references.setPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); + info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -638,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -649,7 +648,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - assert(importedPaths.empty() == 0); // doesn't include possible self reference + assert(importedPaths.size() <= 1); } else { @@ -657,7 +656,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -872,8 +871,8 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drv = evalStore->readDerivation(bfd.drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - const auto drvOutputs = drv.outputsAndOptPaths(*this); - for (auto & output : bfd.outputs) { + auto built = resolveDerivedPath(*this, bfd, &*evalStore); + for (auto & [output, outputPath] : built) { auto outputHash = get(outputHashes, output); if (!outputHash) throw Error( @@ -887,16 +886,11 @@ std::vector<BuildResult> RemoteStore::buildPathsWithResults( throw MissingRealisation(outputId); res.builtOutputs.emplace(realisation->id, *realisation); } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - const auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); res.builtOutputs.emplace( outputId, Realisation { .id = outputId, - .outPath = *drvOutput->second, + .outPath = outputPath, }); } } @@ -920,7 +914,12 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; + BuildResult res { + .path = DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 62daa838c..a1d4daafd 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -53,8 +53,8 @@ public: { return false; } // FIXME extend daemon protocol, move implementation to RemoteStore - std::optional<std::string> getBuildLog(const StorePath & path) override - { unsupported("getBuildLog"); } + std::optional<std::string> getBuildLogExact(const StorePath & path) override + { unsupported("getBuildLogExact"); } private: diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5490df292..c39e50d14 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -200,7 +200,10 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons { assert(info.hash.type == htSHA256); return makeStorePath( - makeType(*this, "text", StoreReferences { info.references }), + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), info.hash, name); } @@ -310,7 +313,7 @@ void Store::addMultipleToStore( bytesExpected += info.narSize; act.setExpected(actCopyPath, bytesExpected); - return info.references.others; + return info.references; }, [&](const StorePath & path) { @@ -815,7 +818,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info->referencesPossiblyToSelf()) + for (auto & j : info->references) s += printStorePath(j) + "\n"; } @@ -877,7 +880,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, { auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->referencesPossiblyToSelf()) + for (auto & ref : info->references) jsonRefs.emplace_back(printStorePath(ref)); } @@ -1205,7 +1208,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.insertReferencePossiblyToSelf(store.parseStorePath(s)); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional<ValidPathInfo>(std::move(info)); @@ -1228,131 +1231,6 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } -StorePathSet ValidPathInfo::referencesPossiblyToSelf() const -{ - return references.possiblyToSelf(path); -} - -void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) -{ - return references.insertPossiblyToSelf(path, std::move(ref)); -} - -void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) -{ - return references.setPossiblyToSelf(path, std::move(refs)); -} - -std::string ValidPathInfo::fingerprint(const Store & store) const -{ - if (narSize == 0) - throw Error("cannot calculate fingerprint of path '%s' because its size is not known", - store.printStorePath(path)); - return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); -} - - -void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) -{ - sigs.insert(secretKey.signDetached(fingerprint(store))); -} - -std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const -{ - if (! ca) - return std::nullopt; - - return StorePathDescriptor { - .name = std::string { path.name() }, - .info = std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { - assert(!references.self); - return TextInfo { - th, - .references = references.others, - }; - }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { - return FixedOutputInfo { - foh, - .references = references, - }; - }, - }, *ca), - }; -} - -bool ValidPathInfo::isContentAddressed(const Store & store) const -{ - auto fullCaOpt = fullStorePathDescriptorOpt(); - - if (! fullCaOpt) - return false; - - auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); - - bool res = caPath == path; - - if (!res) - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); - - return res; -} - - -size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const -{ - if (isContentAddressed(store)) return maxSigs; - - size_t good = 0; - for (auto & sig : sigs) - if (checkSignature(store, publicKeys, sig)) - good++; - return good; -} - - -bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const -{ - return verifyDetached(fingerprint(store), sig, publicKeys); -} - - -Strings ValidPathInfo::shortRefs() const -{ - Strings refs; - for (auto & r : referencesPossiblyToSelf()) - refs.push_back(std::string(r.to_string())); - return refs; -} - - -ValidPathInfo::ValidPathInfo( - const Store & store, - StorePathDescriptor && info, - Hash narHash) - : path(store.makeFixedOutputPathFromCA(info)) - , narHash(narHash) -{ - std::visit(overloaded { - [this](TextInfo && ti) { - this->references = { - .others = std::move(ti.references), - .self = false, - }; - this->ca = std::move((TextHash &&) ti); - }, - [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references); - this->ca = std::move((FixedOutputHash &&) foi); - }, - }, std::move(info.info)); -} - Derivation Store::derivationFromPath(const StorePath & drvPath) { @@ -1372,6 +1250,34 @@ Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool req } } +std::optional<StorePath> Store::getBuildDerivationPath(const StorePath & path) +{ + + if (!path.isDerivation()) { + try { + auto info = queryPathInfo(path); + if (!info->deriver) return std::nullopt; + return *info->deriver; + } catch (InvalidPath &) { + return std::nullopt; + } + } + + if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path)) + return path; + + auto drv = readDerivation(path); + if (!drv.type().hasKnownOutputPaths()) { + // The build log is actually attached to the corresponding + // resolved derivation, so we need to get it first + auto resolvedDrv = drv.tryResolve(*this); + if (resolvedDrv) + return writeDerivation(*this, *resolvedDrv, NoRepair, true); + } + + return path; +} + Derivation Store::readDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, true); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4579be4c6..d77aea338 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -71,6 +71,9 @@ class NarInfoDiskCache; class Store; +typedef std::map<std::string, StorePath> OutputPathMap; + + enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; @@ -122,6 +125,8 @@ public: typedef std::map<std::string, std::string> Params; + + protected: struct PathInfoCacheValue { @@ -614,6 +619,13 @@ public: */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); + /** + * Given a store path, return the realisation actually used in the realisation of this path: + * - If the path is a content-addressed derivation, try to resolve it + * - Otherwise, find one of its derivers + */ + std::optional<StorePath> getBuildDerivationPath(const StorePath &); + /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() @@ -715,6 +727,11 @@ void copyClosure( void removeTempRoots(); +/* Resolve the derived path completely, failing if any derivation output + is unknown. */ +OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); + + /* Return a Store object to access the Nix store denoted by ‘uri’ (slight misnomer...). Supported values are: diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc new file mode 100644 index 000000000..c9c2cafd0 --- /dev/null +++ b/src/libstore/tests/outputs-spec.cc @@ -0,0 +1,187 @@ +#include "outputs-spec.hh" + +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> + +namespace nix { + +#ifndef NDEBUG +TEST(OutputsSpec, no_empty_names) { + ASSERT_DEATH(OutputsSpec::Names { std::set<std::string> { } }, ""); +} +#endif + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(OutputsSpec, bad_ ## NAME) { \ + std::optional OutputsSpecOpt = \ + OutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(OutputsSpecOpt); \ + } + +TEST_DONT_PARSE(empty, "") +TEST_DONT_PARSE(garbage, "&*()") +TEST_DONT_PARSE(double_star, "**") +TEST_DONT_PARSE(star_first, "*,foo") +TEST_DONT_PARSE(star_second, "foo,*") + +#undef TEST_DONT_PARSE + +TEST(OutputsSpec, all) { + std::string_view str = "*"; + OutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out) { + std::string_view str = "out"; + OutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(OutputsSpec::parse(str), expected); + ASSERT_EQ(expected.to_string(), str); +} + +TEST(OutputsSpec, names_out_bin) { + OutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(OutputsSpec::parse("out,bin"), expected); + // N.B. This normalization is OK. + ASSERT_EQ(expected.to_string(), "bin,out"); +} + +#define TEST_SUBSET(X, THIS, THAT) \ + X((OutputsSpec { THIS }).isSubsetOf(THAT)); + +TEST(OutputsSpec, subsets_all_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_all) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, subsets_names_names_eq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, subsets_names_names_noneq) { + TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names { "a" }, (OutputsSpec::Names { "a", "b" })); +} + +TEST(OutputsSpec, not_subsets_all_names) { + TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, not_subsets_names_names) { + TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names { "a", "b" }), (OutputsSpec::Names { "a" })); +} + +#undef TEST_SUBSET + +#define TEST_UNION(RES, THIS, THAT) \ + ASSERT_EQ(OutputsSpec { RES }, (OutputsSpec { THIS }).union_(THAT)); + +TEST(OutputsSpec, union_all_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_all_names) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::All { }, OutputsSpec::Names { "a" }); +} + +TEST(OutputsSpec, union_names_all) { + TEST_UNION(OutputsSpec::All { }, OutputsSpec::Names { "a" }, OutputsSpec::All { }); +} + +TEST(OutputsSpec, union_names_names) { + TEST_UNION((OutputsSpec::Names { "a", "b" }), OutputsSpec::Names { "a" }, OutputsSpec::Names { "b" }); +} + +#undef TEST_UNION + +#define TEST_DONT_PARSE(NAME, STR) \ + TEST(ExtendedOutputsSpec, bad_ ## NAME) { \ + std::optional extendedOutputsSpecOpt = \ + ExtendedOutputsSpec::parseOpt(STR); \ + ASSERT_FALSE(extendedOutputsSpecOpt); \ + } + +TEST_DONT_PARSE(carot_empty, "^") +TEST_DONT_PARSE(prefix_carot_empty, "foo^") +TEST_DONT_PARSE(garbage, "^&*()") +TEST_DONT_PARSE(double_star, "^**") +TEST_DONT_PARSE(star_first, "^*,foo") +TEST_DONT_PARSE(star_second, "^foo,*") + +#undef TEST_DONT_PARSE + +TEST(ExtendedOutputsSpec, defeault) { + std::string_view str = "foo"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = ExtendedOutputsSpec::Default { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, all) { + std::string_view str = "foo^*"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::All { }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out) { + std::string_view str = "foo^out"; + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), str); +} + +TEST(ExtendedOutputsSpec, out_bin) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin"); + ASSERT_EQ(prefix, "foo"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bin,out"); +} + +TEST(ExtendedOutputsSpec, many_carrot) { + auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin"); + ASSERT_EQ(prefix, "foo^bar"); + ExtendedOutputsSpec expected = OutputsSpec::Names { "out", "bin" }; + ASSERT_EQ(extendedOutputsSpec, expected); + ASSERT_EQ(std::string { prefix } + expected.to_string(), "foo^bar^bin,out"); +} + + +#define TEST_JSON(TYPE, NAME, STR, VAL) \ + \ + TEST(TYPE, NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + ((nlohmann::json) TYPE { VAL })); \ + } \ + \ + TEST(TYPE, NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + TYPE { VAL }, \ + (STR ## _json).get<TYPE>()); \ + } + +TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All { }) +TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names { "a" }) +TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names { "a", "b" })) + +TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default { }) +TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit { OutputsSpec::All { } }) +TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a" } }) +TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit { OutputsSpec::Names { "a", "b" } })) + +#undef TEST_JSON + +} diff --git a/src/libstore/tests/path-with-outputs.cc b/src/libstore/tests/path-with-outputs.cc deleted file mode 100644 index 350ea7ffd..000000000 --- a/src/libstore/tests/path-with-outputs.cc +++ /dev/null @@ -1,46 +0,0 @@ -#include "path-with-outputs.hh" - -#include <gtest/gtest.h> - -namespace nix { - -TEST(parseOutputsSpec, basic) -{ - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec)); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^*"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get_if<AllOutputs>(&outputsSpec)); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^out"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin"); - ASSERT_EQ(prefix, "foo"); - ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin"); - ASSERT_EQ(prefix, "foo^bar"); - ASSERT_TRUE(std::get<OutputNames>(outputsSpec) == OutputNames({"out", "bin"})); - } - - { - auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()"); - ASSERT_EQ(prefix, "foo^&*()"); - ASSERT_TRUE(std::get_if<DefaultOutputs>(&outputsSpec)); - } -} - -} diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh new file mode 100644 index 000000000..bd75748ad --- /dev/null +++ b/src/libutil/json-impls.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "nlohmann/json_fwd.hpp" + +// Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types +#define JSON_IMPL(TYPE) \ + namespace nlohmann { \ + using namespace nix; \ + template <> \ + struct adl_serializer<TYPE> { \ + static TYPE from_json(const json & json); \ + static void to_json(json & json, TYPE t); \ + }; \ + } diff --git a/src/libutil/reference-set.hh b/src/libutil/reference-set.hh deleted file mode 100644 index ac4a9994e..000000000 --- a/src/libutil/reference-set.hh +++ /dev/null @@ -1,68 +0,0 @@ -#pragma once - -#include "comparator.hh" - -#include <set> - -namespace nix { - -template<typename Ref> -struct References -{ - std::set<Ref> others; - bool self = false; - - bool empty() const; - size_t size() const; - - /* Functions to view references + self as one set, mainly for - compatibility's sake. */ - std::set<Ref> possiblyToSelf(const Ref & self) const; - void insertPossiblyToSelf(const Ref & self, Ref && ref); - void setPossiblyToSelf(const Ref & self, std::set<Ref> && refs); - - GENERATE_CMP(References<Ref>, me->others, me->self); -}; - -template<typename Ref> -bool References<Ref>::empty() const -{ - return !self && others.empty(); -} - -template<typename Ref> -size_t References<Ref>::size() const -{ - return (self ? 1 : 0) + others.size(); -} - -template<typename Ref> -std::set<Ref> References<Ref>::possiblyToSelf(const Ref & selfRef) const -{ - std::set<Ref> refs { others }; - if (self) - refs.insert(selfRef); - return refs; -} - -template<typename Ref> -void References<Ref>::insertPossiblyToSelf(const Ref & selfRef, Ref && ref) -{ - if (ref == selfRef) - self = true; - else - others.insert(std::move(ref)); -} - -template<typename Ref> -void References<Ref>::setPossiblyToSelf(const Ref & selfRef, std::set<Ref> && refs) -{ - if (refs.count(selfRef)) { - self = true; - refs.erase(selfRef); - } - - others = refs; -} - -} diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 6e325db98..250e83a38 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -312,6 +312,42 @@ namespace nix { } /* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + + TEST(getLine, all) { + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } + } + + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index adcaab686..049838bb1 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -397,7 +397,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { .drvPath = bashDrv, - .outputs = {"out"}, + .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -421,7 +421,7 @@ static void main_nix_build(int argc, char * * argv) { pathsToBuild.push_back(DerivedPath::Built { .drvPath = inputDrv, - .outputs = inputOutputs + .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); } @@ -591,7 +591,7 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 31823a966..406e548c0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -478,9 +478,14 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector<DerivedPath> targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{*drvPath}); + targets.push_back(DerivedPath::Built{ + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); else - targets.push_back(DerivedPath::Opaque{i.queryOutPath()}); + targets.push_back(DerivedPath::Opaque{ + .path = i.queryOutPath(), + }); printMissing(state.store, targets); } @@ -751,8 +756,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) auto drvPath = drv.queryDrvPath(); std::vector<DerivedPath> paths { drvPath - ? (DerivedPath) (DerivedPath::Built { *drvPath }) - : (DerivedPath) (DerivedPath::Opaque { drv.queryOutPath() }), + ? (DerivedPath) (DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }) + : (DerivedPath) (DerivedPath::Opaque { + .path = drv.queryOutPath(), + }), }; printMissing(globals.state->store, paths); if (globals.dryRun) return; diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 36d774dca..577cadceb 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref<Store> store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { + for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index d2eebca7a..425d61e53 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->referencesPossiblyToSelf()) { + for (auto & p : info->references) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5cb5aa53a..5b261ecc6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -263,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); + auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -344,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) + for (auto & p : store->queryPathInfo(j)->references) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->referencesPossiblyToSelf()); + worker_proto::write(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -964,7 +964,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom<StorePathSet> {})); + info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/app.cc b/src/nix/app.cc index 5658f2a52..08cd0ccd4 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -19,12 +19,11 @@ struct InstallableDerivedPath : Installable { } - std::string what() const override { return derivedPath.to_string(*store); } - DerivedPaths toDerivedPaths() override + DerivedPathsWithInfo toDerivedPaths() override { - return {derivedPath}; + return {{derivedPath}}; } std::optional<StorePath> getStorePath() override @@ -80,9 +79,29 @@ UnresolvedApp Installable::toApp(EvalState & state) if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector<StorePathWithOutputs> context2; - for (auto & [path, name] : context) - context2.push_back({path, {name}}); + std::vector<DerivedPath> context2; + for (auto & c : context) { + context2.emplace_back(std::visit(overloaded { + [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { + /* We want all outputs of the drv */ + return DerivedPath::Built { + .drvPath = d.drvPath, + .outputs = OutputsSpec::All {}, + }; + }, + [&](const NixStringContextElem::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + [&](const NixStringContextElem::Opaque & o) -> DerivedPath { + return DerivedPath::Opaque { + .path = o.path, + }; + }, + }, c.raw())); + } return UnresolvedApp{App { .context = std::move(context2), @@ -106,7 +125,10 @@ UnresolvedApp Installable::toApp(EvalState & state) : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { - .context = { { drvPath, {outputName} } }, + .context = { DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName }, + } }, .program = program, }}; } @@ -124,7 +146,7 @@ App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store) for (auto & ctxElt : unresolved.context) installableContext.push_back( - std::make_shared<InstallableDerivedPath>(store, ctxElt.toDerivedPath())); + std::make_shared<InstallableDerivedPath>(store, ctxElt)); auto builtContext = Installable::build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); diff --git a/src/nix/build.cc b/src/nix/build.cc index 94b169167..12b22d999 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -94,13 +94,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (dryRun) { std::vector<DerivedPath> pathsToBuild; - for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); - } + for (auto & i : installables) + for (auto & b : i->toDerivedPaths()) + pathsToBuild.push_back(b.path); + printMissing(store, pathsToBuild, lvlError); + if (json) logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 74a7973b0..6ae9460f6 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -75,10 +75,10 @@ struct CmdBundle : InstallableCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, outputsSpec] = parseFlakeRefWithFragmentAndOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, outputsSpec, + evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec, {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, @@ -105,7 +105,12 @@ struct CmdBundle : InstallableCommand auto outPath = evalState->coerceToStorePath(attr2->pos, *attr2->value, context2, ""); - store->buildPaths({ DerivedPath::Built { drvPath } }); + store->buildPaths({ + DerivedPath::Built { + .drvPath = drvPath, + .outputs = OutputsSpec::All { }, + }, + }); auto outPathS = store->printStorePath(outPath); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1d90d1dac..16bbd8613 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,7 +3,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" @@ -232,7 +232,12 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); + store->buildPaths( + { DerivedPath::Built { + .drvPath = shellDrvPath, + .outputs = OutputsSpec::All { }, + }}, + bmNormal, evalStore); for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9b4cdf35a..d16d88ef8 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,7 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "derivations.hh" -#include "path-with-outputs.hh" +#include "outputs-spec.hh" #include "attr-path.hh" #include "fetchers.hh" #include "registry.hh" @@ -348,7 +348,7 @@ struct CmdFlakeCheck : FlakeCommand // FIXME auto app = App(*state, v); for (auto & i : app.context) { - auto [drvPathS, outputName] = decodeContext(i); + auto [drvPathS, outputName] = NixStringContextElem::parse(i); store->parseStorePath(drvPathS); } #endif @@ -513,8 +513,12 @@ struct CmdFlakeCheck : FlakeCommand auto drvPath = checkDerivation( fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) - drvPaths.push_back(DerivedPath::Built{*drvPath}); + if (drvPath && attr_name == settings.thisSystem.get()) { + drvPaths.push_back(DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); + } } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index 72d02ef11..a0598ca13 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -49,7 +49,7 @@ struct CmdLog : InstallableCommand [&](const DerivedPath::Built & bfd) { return logSub.getBuildLog(bfd.drvPath); }, - }, b.raw()); + }, b.path.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8a0f06435..aac8e5c81 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -22,7 +22,7 @@ struct ProfileElementSource // FIXME: record original attrpath. FlakeRef resolvedRef; std::string attrPath; - OutputsSpec outputs; + ExtendedOutputsSpec outputs; bool operator < (const ProfileElementSource & other) const { @@ -32,17 +32,19 @@ struct ProfileElementSource } }; +const int defaultPriority = 5; + struct ProfileElement { StorePathSet storePaths; std::optional<ProfileElementSource> source; bool active = true; - int priority = 5; + int priority = defaultPriority; std::string describe() const { if (source) - return fmt("%s#%s%s", source->originalRef, source->attrPath, printOutputsSpec(source->outputs)); + return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string()); StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); @@ -124,7 +126,7 @@ struct ProfileManifest parseFlakeRef(e[sOriginalUrl]), parseFlakeRef(e[sUrl]), e["attrPath"], - e["outputs"].get<OutputsSpec>() + e["outputs"].get<ExtendedOutputsSpec>() }; } elements.emplace_back(std::move(element)); @@ -262,13 +264,20 @@ struct ProfileManifest } }; -static std::map<Installable *, BuiltPaths> +static std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> builtPathsPerInstallable( const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths) { - std::map<Installable *, BuiltPaths> res; - for (auto & [installable, builtPath] : builtPaths) - res[installable.get()].push_back(builtPath.path); + std::map<Installable *, std::pair<BuiltPaths, ExtraPathInfo>> res; + for (auto & [installable, builtPath] : builtPaths) { + auto & r = res[installable.get()]; + /* Note that there could be conflicting info + (e.g. meta.priority fields) if the installable returned + multiple derivations. So pick one arbitrarily. FIXME: + print a warning? */ + r.first.push_back(builtPath.path); + r.second = builtPath.info; + } return res; } @@ -308,28 +317,25 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile for (auto & installable : installables) { ProfileElement element; + auto & [res, info] = builtPaths[installable.get()]; - - if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { - // FIXME: make build() return this? - auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + if (info.originalRef && info.resolvedRef && info.attrPath && info.extendedOutputsSpec) { element.source = ProfileElementSource { - installable2->flakeRef, - resolvedRef, - attrPath, - installable2->outputsSpec + .originalRef = *info.originalRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = *info.extendedOutputsSpec, }; - - if(drv.priority) { - element.priority = *drv.priority; - } } - if(priority) { // if --priority was specified we want to override the priority of the installable - element.priority = *priority; - }; + // If --priority was specified we want to override the + // priority of the installable. + element.priority = + priority + ? *priority + : info.priority.value_or(defaultPriority); - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, res); manifest.elements.push_back(std::move(element)); } @@ -487,18 +493,22 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf Strings{}, lockFlags); - auto [attrPath, resolvedRef, drv] = installable->toDerivation(); + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) continue; + auto & info = derivedPaths[0].info; + + assert(info.resolvedRef && info.attrPath); - if (element.source->resolvedRef == resolvedRef) continue; + if (element.source->resolvedRef == info.resolvedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, resolvedRef); + element.source->attrPath, element.source->resolvedRef, *info.resolvedRef); element.source = ProfileElementSource { - installable->flakeRef, - resolvedRef, - attrPath, - installable->outputsSpec + .originalRef = installable->flakeRef, + .resolvedRef = *info.resolvedRef, + .attrPath = *info.attrPath, + .outputs = installable->extendedOutputsSpec, }; installables.push_back(installable); @@ -526,7 +536,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()].first); } updateProfile(manifest.build(store)); @@ -554,8 +564,8 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + printOutputsSpec(element.source->outputs) : "-", + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 2e288f743..d5fab5f2f 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -33,13 +33,7 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand auto dstStore = getDstStore(); auto & dstLogStore = require<LogStore>(*dstStore); - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & drvPath : i->toDrvPaths(getEvalStore())) - drvPaths.insert(drvPath); - - for (auto & drvPath : drvPaths) { + for (auto & drvPath : Installable::toDerivations(getEvalStore(), installables, true)) { if (auto log = srcLogStore.getBuildLog(drvPath)) dstLogStore.addBuildLog(drvPath, *log); else diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 33cd13600..76125e5e4 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -136,7 +136,7 @@ struct CmdWhyDepends : SourceExprCommand for (auto & path : closure) graph.emplace(path, Node { .path = path, - .refs = store->queryPathInfo(path)->references.others, + .refs = store->queryPathInfo(path)->references, .dist = path == dependencyPath ? 0 : inf }); diff --git a/tests/build-hook-ca-floating.nix b/tests/build-hook-ca-floating.nix index 67295985f..dfdbb82da 100644 --- a/tests/build-hook-ca-floating.nix +++ b/tests/build-hook-ca-floating.nix @@ -1,53 +1,6 @@ { busybox }: -with import ./config.nix; - -let - - mkDerivation = args: - derivation ({ - inherit system; - builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - __contentAddressed = true; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; - - input1 = mkDerivation { - shell = busybox; - name = "build-remote-input-1"; - buildCommand = "echo FOO > $out"; - requiredSystemFeatures = ["foo"]; - }; - - input2 = mkDerivation { - shell = busybox; - name = "build-remote-input-2"; - buildCommand = "echo BAR > $out"; - requiredSystemFeatures = ["bar"]; - }; - - input3 = mkDerivation { - shell = busybox; - name = "build-remote-input-3"; - buildCommand = '' - read x < ${input2} - echo $x BAZ > $out - ''; - requiredSystemFeatures = ["baz"]; - }; - -in - - mkDerivation { - shell = busybox; - name = "build-remote"; - buildCommand = - '' - read x < ${input1} - read y < ${input3} - echo "$x $y" > $out - ''; - } +import ./build-hook.nix { + inherit busybox; + contentAddressed = true; +} diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 643334caa..7effd7903 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -1,15 +1,22 @@ -{ busybox }: +{ busybox, contentAddressed ? false }: with import ./config.nix; let + caArgs = if contentAddressed then { + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + __contentAddressed = true; + } else {}; + mkDerivation = args: derivation ({ inherit system; builder = busybox; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - } // removeAttrs args ["builder" "meta" "passthru"]) + } // removeAttrs args ["builder" "meta" "passthru"] + // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; input1 = mkDerivation { diff --git a/tests/build-remote.sh b/tests/build-remote.sh index e73c37ea4..25a482003 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -63,12 +63,9 @@ nix path-info --store $TEST_ROOT/machine3 --all \ | grep builder-build-remote-input-3.sh -# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 -if [[ -z "$CONTENT_ADDRESSED" ]]; then - for i in input1 input3; do - nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i - done -fi +for i in input1 input3; do +nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i +done # Behavior of keep-failed out="$(nix-build 2>&1 failing.nix \ diff --git a/tests/containers.nix b/tests/containers.nix index f31f22cf6..a4856b2df 100644 --- a/tests/containers.nix +++ b/tests/containers.nix @@ -16,7 +16,7 @@ makeTest ({ { virtualisation.writableStore = true; virtualisation.diskSize = 2048; virtualisation.additionalPaths = - [ pkgs.stdenv + [ pkgs.stdenvNoCC (import ./systemd-nspawn.nix { inherit nixpkgs; }).toplevel ]; virtualisation.memorySize = 4096; @@ -38,30 +38,30 @@ makeTest ({ # Test that 'id' gives the expected result in various configurations. # Existing UIDs, sandbox. - host.succeed("nix build --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1") + host.succeed("nix build -v --no-auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-1") host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") # Existing UIDs, no sandbox. - host.succeed("nix build --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2") + host.succeed("nix build -v --no-auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-2") host.succeed("[[ $(cat ./result) = 'uid=30001(nixbld1) gid=30000(nixbld) groups=30000(nixbld)' ]]") # Auto-allocated UIDs, sandbox. - host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-3") host.succeed("[[ $(cat ./result) = 'uid=1000(nixbld) gid=100(nixbld) groups=100(nixbld)' ]]") # Auto-allocated UIDs, no sandbox. - host.succeed("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4") + host.succeed("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-4") host.succeed("[[ $(cat ./result) = 'uid=872415232 gid=30000(nixbld) groups=30000(nixbld)' ]]") # Auto-allocated UIDs, UID range, sandbox. - host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-5 --arg uidRange true") host.succeed("[[ $(cat ./result) = 'uid=0(root) gid=0(root) groups=0(root)' ]]") # Auto-allocated UIDs, UID range, no sandbox. - host.fail("nix build --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") + host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - host.succeed("nix build --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; diff --git a/tests/flakes/build-paths.sh b/tests/flakes/build-paths.sh new file mode 100644 index 000000000..08b4d1763 --- /dev/null +++ b/tests/flakes/build-paths.sh @@ -0,0 +1,66 @@ +source ./common.sh + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake2 + +mkdir -p $flake1Dir $flake2Dir + +writeSimpleFlake $flake2Dir +tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT flake2 +hash=$(nix hash path $flake2Dir) + +dep=$(nix store add-path ./common.sh) + +cat > $flake1Dir/flake.nix <<EOF +{ + inputs.flake2.url = "file://$TEST_ROOT/flake.tar.gz"; + + outputs = { self, flake2 }: { + + a1 = builtins.fetchTarball { + #type = "tarball"; + url = "file://$TEST_ROOT/flake.tar.gz"; + sha256 = "$hash"; + }; + + a2 = ./foo; + + a3 = ./.; + + a4 = self.outPath; + + # FIXME + a5 = self; + + a6 = flake2.outPath; + + # FIXME + a7 = "${flake2}/config.nix"; + + # This is only allowed in impure mode. + a8 = builtins.storePath $dep; + + a9 = "$dep"; + }; +} +EOF + +echo bar > $flake1Dir/foo + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 +[[ $(cat $TEST_ROOT/result) = bar ]] + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 + +nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +[[ -e $TEST_ROOT/result/simple.nix ]] + +nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 +diff common.sh $TEST_ROOT/result + +(! nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9) diff --git a/tests/local.mk b/tests/local.mk index 0c87a0928..e4f1c4e89 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -9,6 +9,7 @@ nix_tests = \ flakes/check.sh \ flakes/unlocked-override.sh \ flakes/absolute-paths.sh \ + flakes/build-paths.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 82ad99402..8182a6a83 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr +libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr diff --git a/tests/setuid.nix b/tests/setuid.nix index 82efd6d54..6784615e4 100644 --- a/tests/setuid.nix +++ b/tests/setuid.nix @@ -15,7 +15,7 @@ makeTest { { virtualisation.writableStore = true; nix.settings.substituters = lib.mkForce [ ]; nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; - virtualisation.additionalPaths = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; + virtualisation.additionalPaths = [ pkgs.stdenvNoCC pkgs.pkgsi686Linux.stdenvNoCC ]; }; testScript = { nodes }: '' |