diff options
151 files changed, 4380 insertions, 1701 deletions
diff --git a/.github/labeler.yml b/.github/labeler.yml index dc502b6d5..fce0d3aeb 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -2,5 +2,22 @@ - doc/manual/* - src/nix/**/*.md +"store": + - src/libstore/store-api.* + - src/libstore/*-store.* + +"fetching": + - src/libfetchers/**/* + +"repl": + - src/libcmd/repl.* + - src/nix/repl.* + +"new-cli": + - src/nix/**/* + "tests": + # Unit tests + - src/*/tests/**/* + # Functional and integration tests - tests/**/* diff --git a/.gitignore b/.gitignore index e326966d6..8ceff4ef2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,9 +19,12 @@ perl/Makefile.config /doc/manual/nix.json /doc/manual/conf-file.json /doc/manual/builtins.json +/doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md +/doc/manual/src/command-ref/experimental-features-shortlist.md +/doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md # /scripts/ @@ -1 +1 @@ -2.15.0
\ No newline at end of file +2.16.0 diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 2826486fb..5066d8278 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,8 +1,8 @@ diff --git a/darwin_stop_world.c b/darwin_stop_world.c -index 3dbaa3fb..36a1d1f7 100644 +index 0468aaec..b348d869 100644 --- a/darwin_stop_world.c +++ b/darwin_stop_world.c -@@ -352,6 +352,7 @@ GC_INNER void GC_push_all_stacks(void) +@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void) int nthreads = 0; word total_size = 0; mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; @@ -10,7 +10,7 @@ index 3dbaa3fb..36a1d1f7 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -407,6 +408,19 @@ GC_INNER void GC_push_all_stacks(void) +@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void) GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); } if (altstack_lo) { @@ -30,6 +30,22 @@ index 3dbaa3fb..36a1d1f7 100644 total_size += altstack_hi - altstack_lo; GC_push_all_stack(altstack_lo, altstack_hi); } +diff --git a/include/gc.h b/include/gc.h +index edab6c22..f2c61282 100644 +--- a/include/gc.h ++++ b/include/gc.h +@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void); + (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) + #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ + ++#if !__APPLE__ ++/* Patch doesn't work on apple */ ++#define NIX_BOEHM_PATCH_VERSION 1 ++#endif ++ + #ifdef __cplusplus + } /* extern "C" */ + #endif diff --git a/pthread_stop_world.c b/pthread_stop_world.c index b5d71e62..aed7b0bf 100644 --- a/pthread_stop_world.c diff --git a/configure.ac b/configure.ac index f1f45f868..bb3f92e4d 100644 --- a/configure.ac +++ b/configure.ac @@ -184,7 +184,7 @@ fi # Look for OpenSSL, a required dependency. FIXME: this is only (maybe) # used by S3BinaryCacheStore. -PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) +PKG_CHECK_MODULES([OPENSSL], [libcrypto >= 1.1.1], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) # Look for libarchive. @@ -289,13 +289,24 @@ PKG_CHECK_MODULES([GTEST], [gtest_main]) # Look for rapidcheck. +AC_ARG_VAR([RAPIDCHECK_HEADERS], [include path of gtest headers shipped by RAPIDCHECK]) # No pkg-config yet, https://github.com/emil-e/rapidcheck/issues/302 AC_LANG_PUSH(C++) AC_SUBST(RAPIDCHECK_HEADERS) [CXXFLAGS="-I $RAPIDCHECK_HEADERS $CXXFLAGS"] +[LIBS="-lrapidcheck -lgtest $LIBS"] AC_CHECK_HEADERS([rapidcheck/gtest.h], [], [], [#include <gtest/gtest.h>]) -dnl No good for C++ libs with mangled symbols -dnl AC_CHECK_LIB([rapidcheck], []) +dnl AC_CHECK_LIB doesn't work for C++ libs with mangled symbols +AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[ + #include <gtest/gtest.h> + #include <rapidcheck/gtest.h> + ]], [[ + return RUN_ALL_TESTS(); + ]]) + ], + [], + [AC_MSG_ERROR([librapidcheck is not found.])]) AC_LANG_POP(C++) fi diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 86f2ca567..d04eecf55 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -10,7 +10,9 @@ let result = '' > **Warning** \ - > This program is **experimental** and its interface is subject to change. + > This program is + > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > and its interface is subject to change. # Name @@ -29,19 +31,18 @@ let showSynopsis = command: args: let - showArgument = arg: "*${arg.label}*" + (if arg ? arity then "" else "..."); + showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' `${command}` [*option*...] ${arguments} ''; - maybeSubcommands = if details ? commands && details.commands != {} - then '' + maybeSubcommands = optionalString (details ? commands && details.commands != {}) + '' where *subcommand* is one of the following: ${subcommands} - '' - else ""; + ''; subcommands = if length categories > 1 then listCategories @@ -63,12 +64,11 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = - if details ? doc - then replaceStrings ["@stores@"] [storeDocs] details.doc - else ""; + maybeDocumentation = optionalString + (details ? doc) + (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = if details.flags == {} then "" else '' + maybeOptions = optionalString (details.flags != {}) '' # Options ${showOptions details.flags toplevel.flags} @@ -78,15 +78,19 @@ let let allOptions = options // commonOptions; showCategory = cat: '' - ${if cat != "" then "**${cat}:**" else ""} + ${optionalString (cat != "") "**${cat}:**"} ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} ''; listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let - shortName = if option ? shortName then "/ `-${option.shortName}`" else ""; - labels = if option ? labels then (concatStringsSep " " (map (s: "*${s}*") option.labels)) else ""; + shortName = optionalString + (option ? shortName) + ("/ `-${option.shortName}`"); + labels = optionalString + (option ? labels) + (concatStringsSep " " (map (s: "*${s}*") option.labels)); in trim '' - `--${name}` ${shortName} ${labels} diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix new file mode 100644 index 000000000..30e211c96 --- /dev/null +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -0,0 +1,9 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + '' + - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + ''; +in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix new file mode 100644 index 000000000..adb94355c --- /dev/null +++ b/doc/manual/generate-xp-features.nix @@ -0,0 +1,11 @@ +with builtins; +with import ./utils.nix; + +let + showExperimentalFeature = name: doc: + squash '' + ## [`${name}`]{#xp-feature-${name}} + + ${doc} + ''; +in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index df941d460..63e7e61e4 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -81,19 +81,20 @@ $(d)/%.8: $(d)/src/command-ref/%.md $(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp + @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -106,6 +107,20 @@ $(d)/conf-file.json: $(bindir)/nix $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix + @rm -rf $@ $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' + @mv $@.tmp $@ + +$(d)/xp-features.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp + @mv $@.tmp $@ + $(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @@ -145,7 +160,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 4b654567f..f783d5908 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -95,9 +95,11 @@ - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index cfa6f8b73..286ddc6d6 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists experimental commands. +This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 9f36929f7..576e5ba0b 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -120,7 +120,8 @@ shell in which to build it: ```console $ nix-shell '<nixpkgs>' -A pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} -[nix-shell]$ cd pan-* +[nix-shell]$ cd $sourceRoot +[nix-shell]$ eval ${patchPhase:-patchPhase} [nix-shell]$ eval ${configurePhase:-configurePhase} [nix-shell]$ eval ${buildPhase:-buildPhase} [nix-shell]$ ./pan/gui/pan diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/contributing/experimental-features.md new file mode 100644 index 000000000..ad5cffa91 --- /dev/null +++ b/doc/manual/src/contributing/experimental-features.md @@ -0,0 +1,95 @@ +This section describes the notion of *experimental features*, and how it fits into the big picture of the development of Nix. + +# What are experimental features? + +Experimental features are considered unstable, which means that they can be changed or removed at any time. +Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). +This allows accessing unstable functionality without unwittingly relying on it. + +Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. +This was a source of confusion and controversy. + +# When should a new feature be marked experimental? + +A change in the Nix codebase should be guarded by an experimental feature flag if it is considered likely to be reverted or adapted in a backwards-incompatible manner after gathering more experience with it in practice. + +Examples: + +- Changes to the Nix language, such as new built-ins, syntactic or semantic changes, etc. +- Changes to the command-line interface + +# Lifecycle of an experimental feature + +Experimental features have to be treated on a case-by-case basis. +However, the standard workflow for an experimental feature is as follows: + +- A new feature is implemented in a *pull request* + - It is guarded by an experimental feature flag that is disabled by default +- The pull request is merged, the *experimental* feature ends up in a release + - Using the feature requires explicitly enabling it, signifying awareness of the potential risks + - Being experimental, the feature can still be changed arbitrarily +- The feature can be *removed* + - The associated experimental feature flag is also removed +- The feature can be declared *stable* + - The associated experimental feature flag is removed + - There should be enough evidence of users having tried the feature, such as feedback, fixed bugs, demonstrations of how it is put to use + - Maintainers must feel confident that: + - The feature is designed and implemented sensibly, that it is fit for purpose + - Potential interactions are well-understood + - Stabilising the feature will not incur an outsized maintenance burden in the future + +The following diagram illustrates the process: + +``` + .------. + | idea | + '------' + | + discussion, design, implementation + | + | .-------. + | | | + v v | + .--------------. review + | pull request | | + '--------------' | + | ^ | | + | | '-------' + .---' '----. + | | + merge user feedback, + | (breaking) changes + | | + '---. .----' + | | + v | + +--------------+ + .---| experimental |----. + | +--------------+ | + | | +decision to stabilise decision against + | keeping the feature + | | + v v + +--------+ +---------+ + | stable | | removed | + +--------+ +---------+ +``` + +# Relation to the RFC process + +Experimental features and [RFCs](https://github.com/NixOS/rfcs/) both allow approaching substantial changes while minimizing the risk. +However they serve different purposes: + +- An experimental feature enables developers to iterate on and deliver a new idea without committing to it or requiring a costly long-running fork. + It is primarily an issue of *implementation*, targeting Nix developers and early testers. +- The goal of an RFC is to make explicit all the implications of a change: + Explain why it is wanted, which new use-cases it enables, which interface changes it requires, etc. + It is primarily an issue of *design* and *communication*, targeting the broader community. + +This means that experimental features and RFCs are orthogonal mechanisms, and can be used independently or together as needed. + +# Currently available experimental features + +{{#include ./experimental-feature-descriptions.md}} diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b56d857d1..a9782be5c 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -15,7 +15,7 @@ Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. [store derivation]: #gloss-store-derivation @@ -54,7 +54,7 @@ invoked, the Nix store can be referred to as a "_local_" or a "_remote_" one: - + A *local store* exists on the filesystem of + + A [local store]{#gloss-local-store} exists on the filesystem of the machine where Nix is invoked. You can use other local stores by passing the `--store` flag to the `nix` command. Local stores can be used for building derivations. @@ -65,17 +65,17 @@ served by the `nix-serve` Perl script. [store]: #gloss-store + [local store]: #gloss-local-store - [chroot store]{#gloss-chroot-store}\ - A local store whose canonical path is anything other than `/nix/store`. + A [local store] whose canonical path is anything other than `/nix/store`. - [binary cache]{#gloss-binary-cache}\ A *binary cache* is a Nix store which uses a different format: its metadata and signatures are kept in `.narinfo` files rather than in a - Nix database. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary - cache](https://cache.nixos.org). + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - [store path]{#gloss-store-path}\ The location of a [store object] in the file system, i.e., an @@ -108,7 +108,7 @@ [fixed-output derivations](#gloss-fixed-output-derivation). - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the Nix database that + A substitute is a command invocation stored in the [Nix database] that describes how to build a store object, bypassing the normal build mechanism (i.e., derivations). Typically, the substitute builds the store object by downloading a pre-built version of the store object @@ -127,6 +127,14 @@ builder can rely on external inputs such as the network or the system time) but the Nix model assumes it. + - Nix database{#gloss-nix-database}\ + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. + + Default location: `/nix/var/nix/db`. + + [Nix database]: #gloss-nix-database + - [Nix expression]{#gloss-nix-expression}\ A high-level description of software packages and compositions thereof. Deploying software using Nix entails writing Nix @@ -175,9 +183,9 @@ - [validity]{#gloss-validity}\ A store path is valid if all [store object]s in its [closure] can be read from the [store]. - For a local store, this means: + For a [local store], this means: - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the Nix database as being valid. + - The store path is listed in the [Nix database] as being valid. - All paths in the store path's [closure] are valid. [validity]: #gloss-validity @@ -217,3 +225,9 @@ [string]: ./language/values.md#type-string [path]: ./language/values.md#type-path [attribute name]: ./language/values.md#attribute-set + + - [experimental feature]{#gloss-experimental-feature}\ + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 3e8c48890..307971434 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -208,12 +208,26 @@ Derivations can declare some infrequently used optional attributes. about converting to and from base-32 notation.) - [`__contentAddressed`]{#adv-attr-__contentAddressed} - If this **experimental** attribute is set to true, then the derivation + > **Warning** + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this attribute, you must enable the + > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: + > + > ``` + > extra-experimental-features = ca-derivations + > ``` + + If this attribute is set to `true`, then the derivation outputs will be stored in a content-addressed location rather than the traditional input-addressed one. - This only has an effect if the `ca-derivations` experimental feature is enabled. - Setting this attribute also requires setting `outputHashMode` and `outputHashAlgo` like for *fixed-output derivations* (see above). + Setting this attribute also requires setting + [`outputHashMode`](#adv-attr-outputHashMode) + and + [`outputHashAlgo`](#adv-attr-outputHashAlgo) + like for *fixed-output derivations* (see above). - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather @@ -307,9 +321,11 @@ Derivations can declare some infrequently used optional attributes. - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ > **Warning** - > This is an experimental feature. + > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). > - > To enable it, add the following to [nix.conf](../command-ref/conf-file.md): + > To use this attribute, you must enable the + > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. + > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` > extra-experimental-features = discard-references diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 651134c25..1d2688ede 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,7 +19,7 @@ to subsequent chapters. channel: ```console - $ nix-env -qaP + $ nix-env --query --available --attr-path nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 nixpkgs.firefox firefox-33.0.2 @@ -31,7 +31,7 @@ to subsequent chapters. 1. Install some packages from the channel: ```console - $ nix-env -iA nixpkgs.hello + $ nix-env --install --attr nixpkgs.hello ``` This should download pre-built packages; it should not build them @@ -49,13 +49,13 @@ to subsequent chapters. 1. Uninstall a package: ```console - $ nix-env -e hello + $ nix-env --uninstall hello ``` 1. You can also test a package without installing it: ```console - $ nix-shell -p hello + $ nix-shell --packages hello ``` This builds or downloads GNU Hello and its dependencies, then drops @@ -76,7 +76,7 @@ to subsequent chapters. ```console $ nix-channel --update nixpkgs - $ nix-env -u '*' + $ nix-env --upgrade '*' ``` The latter command will upgrade each installed package for which @@ -95,5 +95,5 @@ to subsequent chapters. them: ```console - $ nix-collect-garbage -d + $ nix-collect-garbage --delete-old ``` diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md new file mode 100644 index 000000000..133121999 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -0,0 +1,58 @@ +# Release 2.15 (2023-04-11) + +* Commands which take installables on the command line can now read them from the standard input if + passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which + exceed the OS argument limit. + +* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` + or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` + or `--to-sri` to convert a hash to Base64 or SRI format, respectively. + + As the choice of hash formats is no longer binary, the `--base16` flag is also added + to explicitly specify the Base16 format, which is still the default. + +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. + + The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. + Using this is better and more clear than relying on the now-removed `.drv` special handling. + + For example, + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv + ``` + + now gives info about the derivation itself, while + + ```shell-session + $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* + ``` + provides information about each of its outputs. + +* The experimental command `nix describe-stores` has been removed. + +* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). + +* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. + They include all common options that can be specified and common environment variables that affect these commands. + + These pages can be viewed offline with `man` using + + * `man nix-store-<operation>` and `man nix-env-<operation>` + * `nix-store --help --<operation>` and `nix-env --help --<operation>`. + +* Nix when used as a client now checks whether the store (the server) trusts the client. + (The store always had to check whether it trusts the client, but now the client is informed of the store's decision.) + This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. + + `nix store ping` and `nix doctor` now display this information. + +* The new command `nix derivation add` allows adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix derivation show`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index ae159de8f..78ae99f4b 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,41 +1,2 @@ # Release X.Y (202?-??-??) -* Commands which take installables on the command line can now read them from the standard input if - passed the `--stdin` flag. This is primarily useful when you have a large amount of paths which - exceed the OS arg limit. - -* The `nix-hash` command now supports Base64 and SRI. Use the flags `--base64` - or `--sri` to specify the format of output hash as Base64 or SRI, and `--to-base64` - or `--to-sri` to convert a hash to Base64 or SRI format, respectively. - - As the choice of hash formats is no longer binary, the `--base16` flag is also added - to explicitly specify the Base16 format, which is still the default. - -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. - - The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. - Using this is better and more clear than relying on the now-removed `.drv` special handling. - - For example, - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv - ``` - - now gives info about the derivation itself, while - - ```shell-session - $ nix path-info /nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^* - ``` - provides information about each of its outputs. - -* The experimental command `nix describe-stores` has been removed. - -* Nix stores and their settings are now documented in [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md). - -* Documentation for operations of `nix-store` and `nix-env` are now available on separate pages of the manual. - They include all common options that can be specified and common environment variables that affect these commands. - - These pages can be viewed offline with `man` using - - * `man nix-store-<operation>` and `man nix-env-<operation>` - * `nix-store --help --<operation>` and `nix-env --help --<operation>`. diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 5eacce0dd..9043dd8cd 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -5,6 +5,9 @@ rec { concatStrings = concatStringsSep ""; + attrsToList = a: + map (name: { inherit name; value = a.${name}; }) (builtins.attrNames a); + replaceStringsRec = from: to: string: # recursively replace occurrences of `from` with `to` within `string` # example: @@ -39,7 +42,9 @@ rec { filterAttrs = pred: set: listToAttrs (concatMap (name: let v = set.${name}; in if pred name v then [(nameValuePair name v)] else []) (attrNames set)); - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value }: + optionalString = cond: string: if cond then string else ""; + + showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: let result = squash '' - ${if useAnchors @@ -49,10 +54,28 @@ rec { ${indent " " body} ''; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To change this setting, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ${name} = ... + ``` + ''; + # separate body to cleanly handle indentation body = '' ${description} + ${experimentalFeatureNote} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} @@ -71,13 +94,13 @@ rec { else "*machine-specific*"; showAliases = aliases: - if aliases == [] then "" else + optionalString (aliases != []) "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - indent = prefix: s: - concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - in result; + indent = prefix: s: + concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); + showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } @@ -219,6 +219,7 @@ enableParallelBuilding = true; + configureFlags = testConfigureFlags; # otherwise configure fails dontBuild = true; doInstallCheck = true; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index de91dc28d..41ecbbeb4 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -27,8 +27,6 @@ static ref<Store> store() if (!_store) { try { initLibStore(); - loadConfFile(); - settings.lockCPU = false; _store = openStore(); } catch (Error & e) { croak("%s", e.what()); @@ -295,7 +293,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libcmd/command-installable-value.cc b/src/libcmd/command-installable-value.cc index d7581534b..7e0c15eb8 100644 --- a/src/libcmd/command-installable-value.cc +++ b/src/libcmd/command-installable-value.cc @@ -4,8 +4,8 @@ namespace nix { void InstallableValueCommand::run(ref<Store> store, ref<Installable> installable) { - auto installableValue = InstallableValue::require(installable); - run(store, installableValue); + auto installableValue = InstallableValue::require(installable); + run(store, installableValue); } } diff --git a/src/libcmd/editor-for.hh b/src/libcmd/editor-for.hh index f752bd849..c8c4e9d9b 100644 --- a/src/libcmd/editor-for.hh +++ b/src/libcmd/editor-for.hh @@ -5,8 +5,10 @@ namespace nix { -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ +/** + * Helper function to generate args that invoke $EDITOR on + * filename:lineno. + */ Strings editorFor(const Path & file, uint32_t line); } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 30f80edb2..3a7ede4e2 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -22,7 +22,7 @@ InstallableValue::getCursor(EvalState & state) static UsageError nonValueInstallable(Installable & installable) { - return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); + return UsageError("installable '%s' does not correspond to a Nix language value", installable.what()); } InstallableValue & InstallableValue::require(Installable & installable) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 57848a5d3..80c08bf1c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -40,6 +40,7 @@ extern "C" { #include "markdown.hh" #include "local-fs-store.hh" #include "progress-bar.hh" +#include "print.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -425,6 +426,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) } +// FIXME: DRY and match or use the parser static bool isVarName(std::string_view s) { if (s.size() == 0) return false; @@ -894,17 +896,6 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m } -std::ostream & printStringValue(std::ostream & str, const char * string) { - str << "\""; - for (const char * i = string; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else str << *i; - str << "\""; - return str; -} // FIXME: lot of cut&paste from Nix's eval.cc. @@ -922,12 +913,14 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; case nBool: - str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; + str << ANSI_CYAN; + printLiteralBool(str, v.boolean); + str << ANSI_NORMAL; break; case nString: str << ANSI_WARNING; - printStringValue(str, v.string.s); + printLiteralString(str, v.string.s); str << ANSI_NORMAL; break; @@ -964,10 +957,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m sorted.emplace(state->symbols[i.name], i.value); for (auto & i : sorted) { - if (isVarName(i.first)) - str << i.first; - else - printStringValue(str, i.first.c_str()); + printAttributeName(str, i.first); str << " = "; if (seen.count(i.second)) str << "«repeated»"; diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index d0d05b1a1..b2bfb5d04 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -17,7 +17,9 @@ std::pair<Value *, PosIdx> findAlongAttrPath( Bindings & autoArgs, Value & vIn); -/* Heuristic to find the filename and lineno or a nix value. */ +/** + * Heuristic to find the filename and lineno or a nix value. + */ std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what); std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 3fe54408b..31215f880 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -13,7 +13,9 @@ namespace nix { class EvalState; struct Value; -/* Map one attribute name to its value. */ +/** + * Map one attribute name to its value. + */ struct Attr { /* the placement of `name` and `pos` in this struct is important. @@ -37,10 +39,12 @@ static_assert(sizeof(Attr) == 2 * sizeof(uint32_t) + sizeof(Value *), "avoid introducing any padding into Attr if at all possible, and do not " "introduce new fields that need not be present for almost every instance."); -/* Bindings contains all the attributes of an attribute set. It is defined - by its size and its capacity, the capacity being the number of Attr - elements allocated after this structure, while the size corresponds to - the number of elements already inserted in this structure. */ +/** + * Bindings contains all the attributes of an attribute set. It is defined + * by its size and its capacity, the capacity being the number of Attr + * elements allocated after this structure, while the size corresponds to + * the number of elements already inserted in this structure. + */ class Bindings { public: @@ -95,7 +99,9 @@ public: size_t capacity() { return capacity_; } - /* Returns the attributes in lexicographically sorted order. */ + /** + * Returns the attributes in lexicographically sorted order. + */ std::vector<const Attr *> lexicographicOrder(const SymbolTable & symbols) const { std::vector<const Attr *> res; @@ -112,9 +118,11 @@ public: friend class EvalState; }; -/* A wrapper around Bindings that ensures that its always in sorted - order at the end. The only way to consume a BindingsBuilder is to - call finish(), which sorts the bindings. */ +/** + * A wrapper around Bindings that ensures that its always in sorted + * order at the end. The only way to consume a BindingsBuilder is to + * call finish(), which sorts the bindings. + */ class BindingsBuilder { Bindings * bindings; diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c90882edc..46c4999c8 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -110,8 +110,10 @@ public: ref<AttrCursor> getAttr(std::string_view name); - /* Get an attribute along a chain of attrsets. Note that this does - not auto-call functors or functions. */ + /** + * Get an attribute along a chain of attrsets. Note that this does + * not auto-call functors or functions. + */ OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); std::string getString(); @@ -130,7 +132,9 @@ public: Value & forceValue(); - /* Force creation of the .drv file in the Nix store. */ + /** + * Force creation of the .drv file in the Nix store. + */ StorePath forceDerivation(); }; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index f8ddd2acc..a988fa40c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -5,7 +5,9 @@ namespace nix { -/* Note: Various places expect the allocated memory to be zeroed. */ +/** + * Note: Various places expect the allocated memory to be zeroed. + */ [[gnu::always_inline]] inline void * allocBytes(size_t n) { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7f2065656..6668add8c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "filetransfer.hh" #include "function-trace.hh" #include "profiles.hh" +#include "print.hh" #include <algorithm> #include <chrono> @@ -104,18 +105,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << integer; break; case tBool: - str << (boolean ? "true" : "false"); + printLiteralBool(str, boolean); break; case tString: - str << "\""; - for (const char * i = string.s; *i; i++) - if (*i == '\"' || *i == '\\') str << "\\" << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; - else str << *i; - str << "\""; + printLiteralString(str, string.s); break; case tPath: str << path; // !!! escaping? @@ -344,6 +337,22 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } +#if HAVE_BOEHMGC +/* Disable GC while this object lives. Used by CoroutineContext. + * + * Boehm keeps a count of GC_disable() and GC_enable() calls, + * and only enables GC when the count matches. + */ +class BoehmDisableGC { +public: + BoehmDisableGC() { + GC_disable(); + }; + ~BoehmDisableGC() { + GC_enable(); + }; +}; +#endif static bool gcInitialised = false; @@ -368,6 +377,15 @@ void initGC() StackAllocator::defaultAllocator = &boehmGCStackAllocator; + +#if NIX_BOEHM_PATCH_VERSION != 1 + printTalkative("Unpatched BoehmGC, disabling GC inside coroutines"); + /* Used to disable GC when entering coroutines on macOS */ + create_coro_gc_hook = []() -> std::shared_ptr<void> { + return std::make_shared<BoehmDisableGC>(); + }; +#endif + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a1b54951e..b3b985683 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -43,7 +43,10 @@ struct PrimOp struct Env { Env * up; - unsigned short prevWith:14; // nr of levels up to next `with' environment + /** + * Number of of levels up to next `with` environment + */ + unsigned short prevWith:14; enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; Value * values[0]; }; @@ -56,8 +59,10 @@ std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const Stati void copyContext(const Value & v, PathSet & context); -/* Cache for calls to addToStore(); maps source paths to the store - paths. */ +/** + * Cache for calls to addToStore(); maps source paths to the store + * paths. + */ typedef std::map<Path, StorePath> SrcToStore; @@ -69,7 +74,9 @@ typedef std::pair<std::string, std::string> SearchPathElem; typedef std::list<SearchPathElem> SearchPath; -/* Initialise the Boehm GC, if applicable. */ +/** + * Initialise the Boehm GC, if applicable. + */ void initGC(); @@ -144,26 +151,36 @@ public: sOutputSpecified; Symbol sDerivationNix; - /* If set, force copying files to the Nix store even if they - already exist there. */ + /** + * If set, force copying files to the Nix store even if they + * already exist there. + */ RepairFlag repair; - /* The allowed filesystem paths in restricted or pure evaluation - mode. */ + /** + * The allowed filesystem paths in restricted or pure evaluation + * mode. + */ std::optional<PathSet> allowedPaths; Bindings emptyBindings; - /* Store used to materialise .drv files. */ + /** + * Store used to materialise .drv files. + */ const ref<Store> store; - /* Store used to build stuff. */ + /** + * Store used to build stuff. + */ const ref<Store> buildStore; RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; - /* Debugger */ + /** + * Debugger + */ void (* debugRepl)(ref<EvalState> es, const ValMap & extraEnv); bool debugStop; bool debugQuit; @@ -219,7 +236,9 @@ public: private: SrcToStore srcToStore; - /* A cache from path names to parse trees. */ + /** + * A cache from path names to parse trees. + */ #if HAVE_BOEHMGC typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *>>> FileParseCache; #else @@ -227,7 +246,9 @@ private: #endif FileParseCache fileParseCache; - /* A cache from path names to values. */ + /** + * A cache from path names to values. + */ #if HAVE_BOEHMGC typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache; #else @@ -239,17 +260,25 @@ private: std::map<std::string, std::pair<bool, std::string>> searchPathResolved; - /* Cache used by checkSourcePath(). */ + /** + * Cache used by checkSourcePath(). + */ std::unordered_map<Path, Path> resolvedPaths; - /* Cache used by prim_match(). */ + /** + * Cache used by prim_match(). + */ std::shared_ptr<RegexCache> regexCache; #if HAVE_BOEHMGC - /* Allocation cache for GC'd Value objects. */ + /** + * Allocation cache for GC'd Value objects. + */ std::shared_ptr<void *> valueAllocCache; - /* Allocation cache for size-1 Env objects. */ + /** + * Allocation cache for size-1 Env objects. + */ std::shared_ptr<void *> env1AllocCache; #endif @@ -265,47 +294,65 @@ public: SearchPath getSearchPath() { return searchPath; } - /* Allow access to a path. */ + /** + * Allow access to a path. + */ void allowPath(const Path & path); - /* Allow access to a store path. Note that this gets remapped to - the real store path if `store` is a chroot store. */ + /** + * Allow access to a store path. Note that this gets remapped to + * the real store path if `store` is a chroot store. + */ void allowPath(const StorePath & storePath); - /* Allow access to a store path and return it as a string. */ + /** + * Allow access to a store path and return it as a string. + */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /* Check whether access to a path is allowed and throw an error if - not. Otherwise return the canonicalised path. */ + /** + * Check whether access to a path is allowed and throw an error if + * not. Otherwise return the canonicalised path. + */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); - /* When using a diverted store and 'path' is in the Nix store, map - 'path' to the diverted location (e.g. /nix/store/foo is mapped - to /home/alice/my-nix/nix/store/foo). However, this is only - done if the context is not empty, since otherwise we're - probably trying to read from the actual /nix/store. This is - intended to distinguish between import-from-derivation and - sources stored in the actual /nix/store. */ + /** + * When using a diverted store and 'path' is in the Nix store, map + * 'path' to the diverted location (e.g. /nix/store/foo is mapped + * to /home/alice/my-nix/nix/store/foo). However, this is only + * done if the context is not empty, since otherwise we're + * probably trying to read from the actual /nix/store. This is + * intended to distinguish between import-from-derivation and + * sources stored in the actual /nix/store. + */ Path toRealPath(const Path & path, const PathSet & context); - /* Parse a Nix expression from the specified file. */ + /** + * Parse a Nix expression from the specified file. + */ Expr * parseExprFromFile(const Path & path); Expr * parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv); - /* Parse a Nix expression from the specified string. */ + /** + * Parse a Nix expression from the specified string. + */ Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv); Expr * parseExprFromString(std::string s, const Path & basePath); Expr * parseStdin(); - /* Evaluate an expression read from the given file to normal - form. Optionally enforce that the top-level expression is - trivial (i.e. doesn't require arbitrary computation). */ + /** + * Evaluate an expression read from the given file to normal + * form. Optionally enforce that the top-level expression is + * trivial (i.e. doesn't require arbitrary computation). + */ void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); - /* Like `evalFile`, but with an already parsed expression. */ + /** + * Like `evalFile`, but with an already parsed expression. + */ void cacheFile( const Path & path, const Path & resolvedPath, @@ -315,37 +362,52 @@ public: void resetFileCache(); - /* Look up a file in the search path. */ + /** + * Look up a file in the search path. + */ Path findFile(const std::string_view path); Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); - /* If the specified search path element is a URI, download it. */ + /** + * If the specified search path element is a URI, download it. + */ std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); - /* Evaluate an expression to normal form, storing the result in - value `v'. */ + /** + * Evaluate an expression to normal form + * + * @param [out] v The resulting is stored here. + */ void eval(Expr * e, Value & v); - /* Evaluation the expression, then verify that it has the expected - type. */ + /** + * Evaluation the expression, then verify that it has the expected + * type. + */ inline bool evalBool(Env & env, Expr * e); inline bool evalBool(Env & env, Expr * e, const PosIdx pos, std::string_view errorCtx); inline void evalAttrs(Env & env, Expr * e, Value & v, const PosIdx pos, std::string_view errorCtx); - /* If `v' is a thunk, enter it and overwrite `v' with the result - of the evaluation of the thunk. If `v' is a delayed function - application, call the function and overwrite `v' with the - result. Otherwise, this is a no-op. */ + /** + * If `v` is a thunk, enter it and overwrite `v` with the result + * of the evaluation of the thunk. If `v` is a delayed function + * application, call the function and overwrite `v` with the + * result. Otherwise, this is a no-op. + */ inline void forceValue(Value & v, const PosIdx pos); template <typename Callable> inline void forceValue(Value & v, Callable getPos); - /* Force a value, then recursively force list elements and - attributes. */ + /** + * Force a value, then recursively force list elements and + * attributes. + */ void forceValueDeep(Value & v); - /* Force `v', and then verify that it has the expected type. */ + /** + * Force `v`, and then verify that it has the expected type. + */ NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx); NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx); bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -356,7 +418,10 @@ public: inline void forceAttrs(Value & v, Callable getPos, std::string_view errorCtx); inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx); - void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); // either lambda or primop + /** + * @param v either lambda or primop + */ + void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); @@ -367,17 +432,23 @@ public: void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; public: - /* Return true iff the value `v' denotes a derivation (i.e. a - set with attribute `type = "derivation"'). */ + /** + * @return true iff the value `v` denotes a derivation (i.e. a + * set with attribute `type = "derivation"`). + */ bool isDerivation(Value & v); std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true); - /* String coercion. Converts strings, paths and derivations to a - string. If `coerceMore' is set, also converts nulls, integers, - booleans and lists to a string. If `copyToStore' is set, - referenced paths are copied to the Nix store as a side effect. */ + /** + * String coercion. + * + * Converts strings, paths and derivations to a + * string. If `coerceMore` is set, also converts nulls, integers, + * booleans and lists to a string. If `copyToStore` is set, + * referenced paths are copied to the Nix store as a side effect. + */ BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, @@ -385,21 +456,31 @@ public: StorePath copyPathToStore(PathSet & context, const Path & path); - /* Path coercion. Converts strings, paths and derivations to a - path. The result is guaranteed to be a canonicalised, absolute - path. Nothing is copied to the store. */ + /** + * Path coercion. + * + * Converts strings, paths and derivations to a + * path. The result is guaranteed to be a canonicalised, absolute + * path. Nothing is copied to the store. + */ Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); - /* Like coerceToPath, but the result must be a store path. */ + /** + * Like coerceToPath, but the result must be a store path. + */ StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); public: - /* The base environment, containing the builtin functions and - values. */ + /** + * The base environment, containing the builtin functions and + * values. + */ Env & baseEnv; - /* The same, but used during parsing to resolve variables. */ + /** + * The same, but used during parsing to resolve variables. + */ std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private private: @@ -449,8 +530,10 @@ private: public: - /* Do a deep equality test between two values. That is, list - elements and attributes are compared recursively. */ + /** + * Do a deep equality test between two values. That is, list + * elements and attributes are compared recursively. + */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); bool isFunctor(Value & fun); @@ -464,11 +547,15 @@ public: callFunction(fun, 1, args, vRes, pos); } - /* Automatically call a function for which each argument has a - default value or has a binding in the `args' map. */ + /** + * Automatically call a function for which each argument has a + * default value or has a binding in the `args` map. + */ void autoCallFunction(Bindings & args, Value & fun, Value & res); - /* Allocation primitives. */ + /** + * Allocation primitives. + */ inline Value * allocValue(); inline Env & allocEnv(size_t size); @@ -488,10 +575,13 @@ public: void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); - /* Print statistics. */ + /** + * Print statistics. + */ void printStats(); - /* Realise the given context, and return a mapping from the placeholders + /** + * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ [[nodiscard]] StringMap realiseContext(const PathSet & context); @@ -551,11 +641,15 @@ struct DebugTraceStacker { DebugTrace trace; }; -/* Return a string representing the type of the value `v'. */ +/** + * @return A string representing the type of the value `v`. + */ std::string_view showType(ValueType type); std::string showType(const Value & v); -/* If `path' refers to a directory, then append "/default.nix". */ +/** + * If `path` refers to a directory, then append "/default.nix". + */ Path resolveExprPath(Path path); struct InvalidPathError : EvalError diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index b6f710288..c1d1b71e5 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -18,7 +18,8 @@ struct FlakeInput; typedef std::map<FlakeId, FlakeInput> FlakeInputs; -/* FlakeInput is the 'Flake'-level parsed form of the "input" entries +/** + * FlakeInput is the 'Flake'-level parsed form of the "input" entries * in the flake file. * * A FlakeInput is normally constructed by the 'parseFlakeInput' @@ -42,7 +43,12 @@ typedef std::map<FlakeId, FlakeInput> FlakeInputs; struct FlakeInput { std::optional<FlakeRef> ref; - bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + /** + * true = process flake to get outputs + * + * false = (fetched) static source path + */ + bool isFlake = true; std::optional<InputPath> follows; FlakeInputs overrides; }; @@ -56,23 +62,42 @@ struct ConfigFile void apply(); }; -/* The contents of a flake.nix file. */ +/** + * The contents of a flake.nix file. + */ struct Flake { - FlakeRef originalRef; // the original flake specification (by the user) - FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake - FlakeRef lockedRef; // the specific local store result of invoking the fetcher - bool forceDirty = false; // pretend that 'lockedRef' is dirty + /** + * The original flake specification (by the user) + */ + FlakeRef originalRef; + /** + * registry references and caching resolved to the specific underlying flake + */ + FlakeRef resolvedRef; + /** + * the specific local store result of invoking the fetcher + */ + FlakeRef lockedRef; + /** + * pretend that 'lockedRef' is dirty + */ + bool forceDirty = false; std::optional<std::string> description; std::shared_ptr<const fetchers::Tree> sourceInfo; FlakeInputs inputs; - ConfigFile config; // 'nixConfig' attribute + /** + * 'nixConfig' attribute + */ + ConfigFile config; ~Flake(); }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); -/* Fingerprint of a locked flake; used as a cache key. */ +/** + * Fingerprint of a locked flake; used as a cache key. + */ typedef Hash Fingerprint; struct LockedFlake @@ -85,50 +110,72 @@ struct LockedFlake struct LockFlags { - /* Whether to ignore the existing lock file, creating a new one - from scratch. */ + /** + * Whether to ignore the existing lock file, creating a new one + * from scratch. + */ bool recreateLockFile = false; - /* Whether to update the lock file at all. If set to false, if any - change to the lock file is needed (e.g. when an input has been - added to flake.nix), you get a fatal error. */ + /** + * Whether to update the lock file at all. If set to false, if any + * change to the lock file is needed (e.g. when an input has been + * added to flake.nix), you get a fatal error. + */ bool updateLockFile = true; - /* Whether to write the lock file to disk. If set to true, if the - any changes to the lock file are needed and the flake is not - writable (i.e. is not a local Git working tree or similar), you - get a fatal error. If set to false, Nix will use the modified - lock file in memory only, without writing it to disk. */ + /** + * Whether to write the lock file to disk. If set to true, if the + * any changes to the lock file are needed and the flake is not + * writable (i.e. is not a local Git working tree or similar), you + * get a fatal error. If set to false, Nix will use the modified + * lock file in memory only, without writing it to disk. + */ bool writeLockFile = true; - /* Whether to use the registries to lookup indirect flake - references like 'nixpkgs'. */ + /** + * Whether to use the registries to lookup indirect flake + * references like 'nixpkgs'. + */ std::optional<bool> useRegistries = std::nullopt; - /* Whether to apply flake's nixConfig attribute to the configuration */ + /** + * Whether to apply flake's nixConfig attribute to the configuration + */ bool applyNixConfig = false; - /* Whether unlocked flake references (i.e. those without a Git - revision or similar) without a corresponding lock are - allowed. Unlocked flake references with a lock are always - allowed. */ + /** + * Whether unlocked flake references (i.e. those without a Git + * revision or similar) without a corresponding lock are + * allowed. Unlocked flake references with a lock are always + * allowed. + */ bool allowUnlocked = true; - /* Whether to commit changes to flake.lock. */ + /** + * Whether to commit changes to flake.lock. + */ bool commitLockFile = false; - /* The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ + /** + * The path to a lock file to read instead of the `flake.lock` file in the top-level flake + */ std::optional<std::string> referenceLockFilePath; - /* The path to a lock file to write to instead of the `flake.lock` file in the top-level flake */ + /** + * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake + */ std::optional<Path> outputLockFilePath; - /* Flake inputs to be overridden. */ + /** + * Flake inputs to be overridden. + */ std::map<InputPath, FlakeRef> inputOverrides; - /* Flake inputs to be updated. This means that any existing lock - for those inputs will be ignored. */ + /** + * Flake inputs to be updated. This means that any existing lock + * for those inputs will be ignored. + */ std::set<InputPath> inputUpdates; }; diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 23d19adb1..a7c9208c0 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -14,7 +14,8 @@ class Store; typedef std::string FlakeId; -/* A flake reference specifies how to fetch a flake or raw source +/** + * A flake reference specifies how to fetch a flake or raw source * (e.g. from a Git repository). It is created from a URL-like syntax * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local @@ -33,14 +34,17 @@ typedef std::string FlakeId; * be lazy), but the fetcher can be invoked at any time via the * FlakeRef to ensure the store is populated with this input. */ - struct FlakeRef { - /* Fetcher-specific representation of the input, sufficient to - perform the fetch operation. */ + /** + * Fetcher-specific representation of the input, sufficient to + * perform the fetch operation. + */ fetchers::Input input; - /* sub-path within the fetched input that represents this input */ + /** + * sub-path within the fetched input that represents this input + */ Path subdir; bool operator==(const FlakeRef & other) const; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index a74e68c9c..ba2fd46f0 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -234,6 +234,11 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON() == other.toJSON(); } +bool LockFile::operator !=(const LockFile & other) const +{ + return !(*this == other); +} + InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 6512509c5..ba4c0c848 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -16,9 +16,11 @@ typedef std::vector<FlakeId> InputPath; struct LockedNode; -/* A node in the lock file. It has outgoing edges to other nodes (its - inputs). Only the root node has this type; all other nodes have - type LockedNode. */ +/** + * A node in the lock file. It has outgoing edges to other nodes (its + * inputs). Only the root node has this type; all other nodes have + * type LockedNode. + */ struct Node : std::enable_shared_from_this<Node> { typedef std::variant<ref<LockedNode>, InputPath> Edge; @@ -28,7 +30,9 @@ struct Node : std::enable_shared_from_this<Node> virtual ~Node() { } }; -/* A non-root node in the lock file. */ +/** + * A non-root node in the lock file. + */ struct LockedNode : Node { FlakeRef lockedRef, originalRef; @@ -63,10 +67,15 @@ struct LockFile void write(const Path & path) const; - /* Check whether this lock file has any unlocked inputs. */ + /** + * Check whether this lock file has any unlocked inputs. + */ std::optional<FlakeRef> isUnlocked() const; bool operator ==(const LockFile & other) const; + // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 + // that is still the default on aarch64-linux) + bool operator !=(const LockFile & other) const; std::shared_ptr<Node> findInput(const InputPath & path); @@ -74,7 +83,9 @@ struct LockFile static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); - /* Check that every 'follows' input target exists. */ + /** + * Check that every 'follows' input target exists. + */ void check(); }; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 51ef7782a..584d64ac1 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -26,7 +26,10 @@ private: mutable std::string outputName; Outputs outputs; - bool failed = false; // set if we get an AssertionError + /** + * Set if we get an AssertionError + */ + bool failed = false; Bindings * attrs = nullptr, * meta = nullptr; @@ -35,7 +38,10 @@ private: bool checkMeta(Value & v); public: - std::string attrPath; /* path towards the derivation */ + /** + * path towards the derivation + */ + std::string attrPath; DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); @@ -47,8 +53,10 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the unordered map of output names to (optional) output paths. - * The "outputs to install" are determined by `meta.outputsToInstall`. */ + /** + * Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. + */ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -80,8 +88,10 @@ typedef std::list<DrvInfo> DrvInfos; #endif -/* If value `v' denotes a derivation, return a DrvInfo object - describing it. Otherwise return nothing. */ +/** + * If value `v` denotes a derivation, return a DrvInfo object + * describing it. Otherwise return nothing. + */ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index eb6f062b4..1557cbbeb 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -3,6 +3,7 @@ #include "eval.hh" #include "symbol-table.hh" #include "util.hh" +#include "print.hh" #include <cstdlib> @@ -60,45 +61,12 @@ Pos::operator std::shared_ptr<AbstractPos>() const return pos; } -/* Displaying abstract syntax trees. */ - -static void showString(std::ostream & str, std::string_view s) -{ - str << '"'; - for (auto c : s) - if (c == '"' || c == '\\' || c == '$') str << "\\" << c; - else if (c == '\n') str << "\\n"; - else if (c == '\r') str << "\\r"; - else if (c == '\t') str << "\\t"; - else str << c; - str << '"'; -} - +// FIXME: remove, because *symbols* are abstract and do not have a single +// textual representation; see printIdentifier() std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) { std::string_view s = symbol; - - if (s.empty()) - str << "\"\""; - else if (s == "if") // FIXME: handle other keywords - str << '"' << s << '"'; - else { - char c = s[0]; - if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { - showString(str, s); - return str; - } - for (auto c : s) - if (!((c >= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - c == '_' || c == '\'' || c == '-')) { - showString(str, s); - return str; - } - str << s; - } - return str; + return printIdentifier(str, s); } void Expr::show(const SymbolTable & symbols, std::ostream & str) const @@ -118,7 +86,7 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - showString(str, s); + printLiteralString(str, s); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 4079a7b24..c2f817c9a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -22,7 +22,9 @@ MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); -/* Position objects. */ +/** + * Position objects. + */ struct Pos { uint32_t line; @@ -133,7 +135,9 @@ class EvalState; struct StaticEnv; -/* An attribute path is a sequence of attribute names. */ +/** + * An attribute path is a sequence of attribute names. + */ struct AttrName { Symbol symbol; @@ -213,11 +217,11 @@ struct ExprVar : Expr or function argument) or from a "with". */ bool fromWith; - /* In the former case, the value is obtained by going `level' + /* In the former case, the value is obtained by going `level` levels up from the current environment and getting the - `displ'th value in that environment. In the latter case, the - value is obtained by getting the attribute named `name' from - the set stored in the environment that is `level' levels up + `displ`th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name` from + the set stored in the environment that is `level` levels up from the current one.*/ Level level; Displacement displ; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f1bce2fb6..510f674eb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1293,7 +1293,13 @@ drvName, Bindings * attrs, Value & v) auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(method, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput::CAFixed { @@ -2103,7 +2109,13 @@ static void addPath( std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = *expectedHash, + }, + .references = {}, + }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { StorePath dstPath = settings.readOnlyMode diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 1c5ce219f..4ae73fe1f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -23,9 +23,11 @@ struct RegisterPrimOp typedef std::vector<Info> PrimOps; static PrimOps * primOps; - /* You can register a constant by passing an arity of 0. fun - will get called during EvalState initialization, so there - may be primops not yet added and builtins is not yet sorted. */ + /** + * You can register a constant by passing an arity of 0. fun + * will get called during EvalState initialization, so there + * may be primops not yet added and builtins is not yet sorted. + */ RegisterPrimOp( std::string name, size_t arity, @@ -38,10 +40,14 @@ struct RegisterPrimOp may wish to use them in limited contexts without globally enabling them. */ -/* Load a ValueInitializer from a DSO and return whatever it initializes */ +/** + * Load a ValueInitializer from a DSO and return whatever it initializes + */ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Value & v); -/* Execute a program and parse its output */ +/** + * Execute a program and parse its output + */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 0d0e00fa5..2e150c9d0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -243,10 +243,15 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // early exit if pinned and already in the store if (expectedHash && expectedHash->type == htSHA256) { - auto expectedPath = - unpack - ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) - : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + auto expectedPath = state.store->makeFixedOutputPath( + name, + FixedOutputInfo { + .hash = { + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, + }, + .references = {} + }); if (state.store->isValidPath(expectedPath)) { state.allowAndSetStorePathString(expectedPath, v); diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc new file mode 100644 index 000000000..d08672cfc --- /dev/null +++ b/src/libexpr/print.cc @@ -0,0 +1,78 @@ +#include "print.hh" + +namespace nix { + +std::ostream & +printLiteralString(std::ostream & str, const std::string_view string) +{ + str << "\""; + for (auto i = string.begin(); i != string.end(); ++i) { + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; + else str << *i; + } + str << "\""; + return str; +} + +std::ostream & +printLiteralBool(std::ostream & str, bool boolean) +{ + str << (boolean ? "true" : "false"); + return str; +} + +std::ostream & +printIdentifier(std::ostream & str, std::string_view s) { + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + printLiteralString(str, s); + return str; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + printLiteralString(str, s); + return str; + } + str << s; + } + return str; +} + +// FIXME: keywords +static bool isVarName(std::string_view s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + +std::ostream & +printAttributeName(std::ostream & str, std::string_view name) { + if (isVarName(name)) + str << name; + else + printLiteralString(str, name); + return str; +} + + +} diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh new file mode 100644 index 000000000..f9cfc3964 --- /dev/null +++ b/src/libexpr/print.hh @@ -0,0 +1,48 @@ +#pragma once +/** + * @file + * @brief Common printing functions for the Nix language + * + * While most types come with their own methods for printing, they share some + * functions that are placed here. + */ + +#include <iostream> + +namespace nix { + /** + * Print a string as a Nix string literal. + * + * Quotes and fairly minimal escaping are added. + * + * @param s The logical string + */ + std::ostream & printLiteralString(std::ostream & o, std::string_view s); + inline std::ostream & printLiteralString(std::ostream & o, const char * s) { + return printLiteralString(o, std::string_view(s)); + } + inline std::ostream & printLiteralString(std::ostream & o, const std::string & s) { + return printLiteralString(o, std::string_view(s)); + } + + /** Print `true` or `false`. */ + std::ostream & printLiteralBool(std::ostream & o, bool b); + + /** + * Print a string as an attribute name in the Nix expression language syntax. + * + * Prints a quoted string if necessary. + */ + std::ostream & printAttributeName(std::ostream & o, std::string_view s); + + /** + * Print a string as an identifier in the Nix expression language syntax. + * + * FIXME: "identifier" is ambiguous. Identifiers do not have a single + * textual representation. They can be used in variable references, + * let bindings, left-hand sides or attribute names in a select + * expression, or something else entirely, like JSON. Use one of the + * `print*` functions instead. + */ + std::ostream & printIdentifier(std::ostream & o, std::string_view s); +} diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c97a0a2db..967a186dd 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -10,15 +10,11 @@ namespace nix { -/* Symbol table used by the parser and evaluator to represent and look - up identifiers and attributes efficiently. SymbolTable::create() - converts a string into a symbol. Symbols have the property that - they can be compared efficiently (using an equality test), - because the symbol table stores only one copy of each string. */ - -/* This class mainly exists to give us an operator<< for ostreams. We could also - return plain strings from SymbolTable, but then we'd have to wrap every - instance of a symbol that is fmt()ed, which is inconvenient and error-prone. */ +/** + * This class mainly exists to give us an operator<< for ostreams. We could also + * return plain strings from SymbolTable, but then we'd have to wrap every + * instance of a symbol that is fmt()ed, which is inconvenient and error-prone. + */ class SymbolStr { friend class SymbolTable; @@ -47,6 +43,11 @@ public: friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); }; +/** + * Symbols have the property that they can be compared efficiently + * (using an equality test), because the symbol table stores only one + * copy of each string. + */ class Symbol { friend class SymbolTable; @@ -66,6 +67,10 @@ public: bool operator!=(const Symbol other) const { return id != other.id; } }; +/** + * Symbol table used by the parser and evaluator to represent and look + * up identifiers and attributes efficiently. + */ class SymbolTable { private: @@ -74,6 +79,9 @@ private: public: + /** + * converts a string into a symbol. + */ Symbol create(std::string_view s) { // Most symbols are looked up more than once, so we trade off insertion performance diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index bfae4ee94..7739f99df 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -36,9 +36,11 @@ typedef enum { tFloat } InternalType; -// This type abstracts over all actual value types in the language, -// grouping together implementation details like tList*, different function -// types, and types in non-normal form (so thunks and co.) +/** + * This type abstracts over all actual value types in the language, + * grouping together implementation details like tList*, different function + * types, and types in non-normal form (so thunks and co.) + */ typedef enum { nThunk, nInt, @@ -70,38 +72,51 @@ class XMLWriter; typedef int64_t NixInt; typedef double NixFloat; -/* External values must descend from ExternalValueBase, so that +/** + * External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented */ class ExternalValueBase { friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); protected: - /* Print out the value */ + /** + * Print out the value + */ virtual std::ostream & print(std::ostream & str) const = 0; public: - /* Return a simple string describing the type */ + /** + * Return a simple string describing the type + */ virtual std::string showType() const = 0; - /* Return a string to be used in builtins.typeOf */ + /** + * Return a string to be used in builtins.typeOf + */ virtual std::string typeOf() const = 0; - /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + /** + * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; - /* Compare to another value of the same type. Defaults to uncomparable, + /** + * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ virtual bool operator ==(const ExternalValueBase & b) const; - /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + /** + * Print the value as JSON. Defaults to unconvertable, i.e. throws an error + */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, PathSet & context, bool copyToStore = true) const; - /* Print the value as XML. Defaults to unevaluated */ + /** + * Print the value as XML. Defaults to unevaluated + */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, const PosIdx pos) const; @@ -146,26 +161,28 @@ public: NixInt integer; bool boolean; - /* Strings in the evaluator carry a so-called `context' which - is a list of strings representing store paths. This is to - allow users to write things like + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like - "--with-freetype2-library=" + freetype + "/lib" + * "--with-freetype2-library=" + freetype + "/lib" - where `freetype' is a derivation (or a source to be copied - to the store). If we just concatenated the strings without - keeping track of the referenced store paths, then if the - string is used as a derivation attribute, the derivation - will not have the correct dependencies in its inputDrvs and - inputSrcs. + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. - The semantics of the context is as follows: when a string - with context C is used as a derivation attribute, then the - derivations in C will be added to the inputDrvs of the - derivation, and the other store paths in C will be added to - the inputSrcs of the derivations. + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. - For canonicity, the store paths should be in sorted order. */ + * For canonicity, the store paths should be in sorted order. + */ struct { const char * s; const char * * context; // must be in sorted order @@ -197,8 +214,10 @@ public: NixFloat fpoint; }; - // Returns the normal type of a Value. This only returns nThunk if the - // Value hasn't been forceValue'd + /** + * Returns the normal type of a Value. This only returns nThunk if + * the Value hasn't been forceValue'd + */ inline ValueType type() const { switch (internalType) { @@ -217,8 +236,10 @@ public: abort(); } - /* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ + /** + * After overwriting an app node, be sure to clear pointers in the + * Value to ensure that the target isn't kept alive unnecessarily. + */ inline void clearValue() { app.left = app.right = 0; @@ -366,9 +387,11 @@ public: PosIdx determinePos(const PosIdx pos) const; - /* Check whether forcing this value requires a trivial amount of - computation. In particular, function applications are - non-trivial. */ + /** + * Check whether forcing this value requires a trivial amount of + * computation. In particular, function applications are + * non-trivial. + */ bool isTrivial() const; NixStringContext getContext(const Store &); @@ -414,7 +437,9 @@ typedef std::map<Symbol, ValueVector> ValueVectorMap; #endif -/* A value allocated in traceable memory. */ +/** + * A value allocated in traceable memory. + */ typedef std::shared_ptr<Value *> RootValue; RootValue allocRootValue(Value * v); diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index d467b4f1d..8719602d8 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -28,34 +28,37 @@ public: class Store; -/* Plain opaque path to some store object. - - Encoded as just the path: ‘<path>’. -*/ +/** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘<path>’. + */ struct NixStringContextElem_Opaque { StorePath path; GENERATE_CMP(NixStringContextElem_Opaque, me->path); }; -/* Path to a derivation and its entire build closure. - - The path doesn't just refer to derivation itself and its closure, but - also all outputs of all derivations in that closure (including the - root derivation). - - Encoded in the form ‘=<drvPath>’. -*/ +/** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=<drvPath>’. + */ struct NixStringContextElem_DrvDeep { StorePath drvPath; GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); }; -/* Derivation output. - - Encoded in the form ‘!<output>!<drvPath>’. -*/ +/** + * Derivation output. + * + * Encoded in the form ‘!<output>!<drvPath>’. + */ struct NixStringContextElem_Built { StorePath drvPath; std::string output; @@ -84,11 +87,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { return static_cast<Raw &>(*this); } - /* Decode a context string, one of: - - ‘<path>’ - - ‘=<path>’ - - ‘!<name>!<path>’ - */ + /** + * Decode a context string, one of: + * - ‘<path>’ + * - ‘=<path>’ + * - ‘!<name>!<path>’ + */ static NixStringContextElem parse(const Store & store, std::string_view s); std::string to_string(const Store & store) const; }; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c767e72e5..91db3a9eb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); + return store.makeFixedOutputPath(getName(), FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + .references = {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index acdecea57..498ad7e4d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -21,14 +21,14 @@ struct Tree struct InputScheme; -/* The Input object is generated by a specific fetcher, based on the +/** + * The Input object is generated by a specific fetcher, based on the * user-supplied input attribute in the flake.nix file, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the * "fromURL()" or "fromAttrs()" static functions which are provided * the url or attrset specified in the flake file. */ - struct Input { friend struct InputScheme; @@ -38,7 +38,9 @@ struct Input bool locked = false; bool direct = true; - /* path of the parent of this input, used for relative path resolution */ + /** + * path of the parent of this input, used for relative path resolution + */ std::optional<Path> parent; public: @@ -56,27 +58,35 @@ public: Attrs toAttrs() const; - /* Check whether this is a "direct" input, that is, not - one that goes through a registry. */ + /** + * Check whether this is a "direct" input, that is, not + * one that goes through a registry. + */ bool isDirect() const { return direct; } - /* Check whether this is a "locked" input, that is, - one that contains a commit hash or content hash. */ + /** + * Check whether this is a "locked" input, that is, + * one that contains a commit hash or content hash. + */ bool isLocked() const { return locked; } - /* Check whether the input carries all necessary info required - for cache insertion and substitution. - These fields are used to uniquely identify cached trees - within the "tarball TTL" window without necessarily - indicating that the input's origin is unchanged. */ + /** + * Check whether the input carries all necessary info required + * for cache insertion and substitution. + * These fields are used to uniquely identify cached trees + * within the "tarball TTL" window without necessarily + * indicating that the input's origin is unchanged. + */ bool hasAllInfo() const; bool operator ==(const Input & other) const; bool contains(const Input & other) const; - /* Fetch the input into the Nix store, returning the location in - the Nix store and the locked input. */ + /** + * Fetch the input into the Nix store, returning the location in + * the Nix store and the locked input. + */ std::pair<Tree, Input> fetch(ref<Store> store) const; Input applyOverrides( @@ -105,7 +115,8 @@ public: }; -/* The InputScheme represents a type of fetcher. Each fetcher +/** + * The InputScheme represents a type of fetcher. Each fetcher * registers with nix at startup time. When processing an input for a * flake, each scheme is given an opportunity to "recognize" that * input from the url or attributes in the flake file's specification diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e9686262a..96fe5faca 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,15 +71,19 @@ DownloadFileResult downloadFile( dumpString(res.data, sink); auto hash = hashString(htSHA256, res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + name, + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + .references = {}, + }, hashString(htSHA256, sink.s), }; info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; - auto source = StringSource(sink.s); + auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 37664c065..56f47a4ac 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -10,7 +10,6 @@ #include <cctype> #include <exception> #include <iostream> -#include <mutex> #include <cstdlib> #include <sys/time.h> @@ -20,16 +19,9 @@ #ifdef __linux__ #include <features.h> #endif -#ifdef __GLIBC__ -#include <gnu/lib-names.h> -#include <nss.h> -#include <dlfcn.h> -#endif #include <openssl/crypto.h> -#include <sodium.h> - namespace nix { @@ -115,57 +107,6 @@ std::string getArg(const std::string & opt, return *i; } - -#if OPENSSL_VERSION_NUMBER < 0x10101000L -/* OpenSSL is not thread-safe by default - it will randomly crash - unless the user supplies a mutex locking function. So let's do - that. */ -static std::vector<std::mutex> opensslLocks; - -static void opensslLockCallback(int mode, int type, const char * file, int line) -{ - if (mode & CRYPTO_LOCK) - opensslLocks[type].lock(); - else - opensslLocks[type].unlock(); -} -#endif - -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { -#ifdef __GLIBC__ - /* On linux, glibc will run every lookup through the nss layer. - * That means every lookup goes, by default, through nscd, which acts as a local - * cache. - * Because we run builds in a sandbox, we also remove access to nscd otherwise - * lookups would leak into the sandbox. - * - * But now we have a new problem, we need to make sure the nss_dns backend that - * does the dns lookups when nscd is not available is loaded or available. - * - * We can't make it available without leaking nix's environment, so instead we'll - * load the backend, and configure nss so it does not try to run dns lookups - * through nscd. - * - * This is technically only used for builtins:fetch* functions so we only care - * about dns. - * - * All other platforms are unaffected. - */ - if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) - warn("unable to load nss_dns backend"); - // FIXME: get hosts entry from nsswitch.conf. - __nss_configure_lookup("hosts", "files dns"); -#endif - }); -} - static void sigHandler(int signo) { } @@ -177,16 +118,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif -#if OPENSSL_VERSION_NUMBER < 0x10101000L - /* Initialise OpenSSL locking. */ - opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); - CRYPTO_set_locking_callback(opensslLockCallback); -#endif - - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - loadConfFile(); + initLibStore(); startSignalHandlerThread(); @@ -223,7 +155,10 @@ void initNix() if (sigaction(SIGTRAP, &act, 0)) throw SysError("handling SIGTRAP"); #endif - /* Register a SIGSEGV handler to detect stack overflows. */ + /* Register a SIGSEGV handler to detect stack overflows. + Why not initLibExpr()? initGC() is essentially that, but + detectStackOverflow is not an instance of the init function concept, as + it may have to be invoked more than once per process. */ detectStackOverflow(); /* There is no privacy in the Nix system ;-) At least not for @@ -236,16 +171,6 @@ void initNix() gettimeofday(&tv, 0); srandom(tv.tv_usec); - /* On macOS, don't use the per-session TMPDIR (as set e.g. by - sshd). This breaks build users because they don't have access - to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ - if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) - unsetenv("TMPDIR"); -#endif - - preloadNSS(); - initLibStore(); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index d915a4a65..7a9e83c6c 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -25,7 +25,9 @@ public: int handleExceptions(const std::string & programName, std::function<void()> fun); -/* Don't forget to call initPlugins() after settings are initialized! */ +/** + * Don't forget to call initPlugins() after settings are initialized! + */ void initNix(); void parseCmdLine(int argc, char * * argv, @@ -36,7 +38,9 @@ void parseCmdLine(const std::string & programName, const Strings & args, void printVersion(const std::string & programName); -/* Ugh. No better place to put this. */ +/** + * Ugh. No better place to put this. + */ void printGCWarning(); class Store; @@ -75,11 +79,15 @@ struct LegacyArgs : public MixCommonArgs }; -/* Show the manual page for the specified program. */ +/** + * Show the manual page for the specified program. + */ void showManPage(const std::string & name); -/* The constructor of this class starts a pager if stdout is a - terminal and $PAGER is set. Stdout is redirected to the pager. */ +/** + * The constructor of this class starts a pager if stdout is a + * terminal and $PAGER is set. Stdout is redirected to the pager. + */ class RunPager { public: @@ -110,28 +118,34 @@ struct PrintFreed }; -/* Install a SIGSEGV handler to detect stack overflows. */ +/** + * Install a SIGSEGV handler to detect stack overflows. + */ void detectStackOverflow(); -/* Pluggable behavior to run in case of a stack overflow. - - Default value: defaultStackOverflowHandler. - - This is called by the handler installed by detectStackOverflow(). - - This gives Nix library consumers a limit opportunity to report the error - condition. The handler should exit the process. - See defaultStackOverflowHandler() for a reference implementation. - - NOTE: Use with diligence, because this runs in the signal handler, with very - limited stack space and a potentially a corrupted heap, all while the failed - thread is blocked indefinitely. All functions called must be reentrant. */ +/** + * Pluggable behavior to run in case of a stack overflow. + * + * Default value: defaultStackOverflowHandler. + * + * This is called by the handler installed by detectStackOverflow(). + * + * This gives Nix library consumers a limit opportunity to report the error + * condition. The handler should exit the process. + * See defaultStackOverflowHandler() for a reference implementation. + * + * NOTE: Use with diligence, because this runs in the signal handler, with very + * limited stack space and a potentially a corrupted heap, all while the failed + * thread is blocked indefinitely. All functions called must be reentrant. + */ extern std::function<void(siginfo_t * info, void * ctx)> stackOverflowHandler; -/* The default, robust implementation of stackOverflowHandler. - - Prints an error message directly to stderr using a syscall instead of the - logger. Exits the process immediately after. */ +/** + * The default, robust implementation of stackOverflowHandler. + * + * Prints an error message directly to stderr using a syscall instead of the + * logger. Exits the process immediately after. + */ void defaultStackOverflowHandler(siginfo_t * info, void * ctx); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 751cf8c30..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -306,11 +306,22 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name, references), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = nar.first, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; return info; })->path; } @@ -414,15 +425,22 @@ StorePath BinaryCacheStore::addToStore( }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name, references), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -434,7 +452,7 @@ StorePath BinaryCacheStore::addTextToStore( RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) return path; @@ -443,10 +461,16 @@ StorePath BinaryCacheStore::addTextToStore( dumpString(s, sink); StringSource source(sink.s); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + std::string { name }, + TextInfo { + { .hash = textHash }, + references, + }, + nar.first, + }; info.narSize = nar.second; - info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5e52d7844..49f271d24 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -46,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore @@ -75,14 +80,15 @@ public: std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback<std::optional<std::string>> callback) noexcept; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a12c599d9..27d1a1b6c 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -12,9 +12,12 @@ namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -22,8 +25,10 @@ struct BuildResult PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -33,7 +38,12 @@ struct BuildResult NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -60,27 +70,39 @@ struct BuildResult return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ + /** + * The derivation we built or the store path we substituted. + */ DerivedPath path; - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ DrvOutputs builtOutputs; - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; - /* User and system CPU time the build took. */ + /** + * User and system CPU time the build took. + */ std::optional<std::chrono::microseconds> cpuUser, cpuSystem; bool success() diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index f43ce22af..3a6f0c2d9 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -16,8 +16,10 @@ struct HookInstance; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -27,11 +29,15 @@ enum struct PathStatus { struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -46,59 +52,87 @@ struct InitialOutput { struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr<DerivationGoal> resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ + /** + * The specific outputs that we need to build. Empty means all of + * them. + */ OutputsSpec wantedOutputs; - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ + /** + * Whether additional wanted outputs have been added. + */ bool needRestart = false; - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ bool retrySubstitution = false; - /* Whether we've retried substitution, in which case we won't try - again. */ + /** + * Whether we've retried substitution, in which case we won't try + * again. + */ bool retriedSubstitution = false; - /* The derivation stored at drvPath. */ + /** + * The derivation stored at drvPath. + */ std::unique_ptr<Derivation> drv; std::unique_ptr<ParsedDerivation> parsedDrv; - /* The remainder is state held during the build. */ + /** + * The remainder is state held during the build. + */ - /* Locks on (fixed) output paths. */ + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map<std::string, InitialOutput> initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr<BufferedSink> logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list<std::string> logTail; std::string currentLogLine; @@ -106,10 +140,14 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* The build hook. */ + /** + * The build hook. + */ std::unique_ptr<HookInstance> hook; - /* The sort of derivation we are building. */ + /** + * The sort of derivation we are building. + */ DerivationType derivationType; typedef void (DerivationGoal::*GoalState)(); @@ -121,12 +159,16 @@ struct DerivationGoal : public Goal std::unique_ptr<Activity> act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr<Activity> actLock; std::map<ActivityId, Activity> builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, @@ -143,10 +185,14 @@ struct DerivationGoal : public Goal void work() override; - /* Add wanted outputs to an already existing derivation goal. */ + /** + * Add wanted outputs to an already existing derivation goal. + */ void addWantedOutputs(const OutputsSpec & outputs); - /* The states. */ + /** + * The states. + */ void getDerivation(); void loadDerivation(); void haveDerivation(); @@ -160,28 +206,42 @@ struct DerivationGoal : public Goal void resolvedFinished(); - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ virtual DrvOutputs registerOutputs(); - /* Open a log file and a pipe to it. */ + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -191,30 +251,40 @@ struct DerivationGoal : public Goal virtual bool isReadDesc(int fd); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'DrvOutputs' structure containing the valid and wanted + * outputs. + */ std::pair<bool, DrvOutputs> checkPathValidity(); - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'DrvOutputs' structure containing the wanted + * outputs. + */ DrvOutputs assertPathValidity(); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); void repairClosure(); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 3b6620b76..697ddb283 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -11,24 +11,34 @@ namespace nix { class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { - // The drv output we're trying to substitue + /** + * The drv output we're trying to substitue + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ std::shared_ptr<const Realisation> outputInfo; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list<ref<Store>> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr<Store> sub; struct DownloadState @@ -39,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal { std::shared_ptr<DownloadState> downloadState; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; public: diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 924a8bbd5..f4bf6f38b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -7,11 +7,15 @@ namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; @@ -19,48 +23,72 @@ struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set<GoalPtr, CompareGoalPtrs> Goals; typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; struct Goal : public std::enable_shared_from_this<Goal> { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ + /** + * Build result. + */ BuildResult buildResult; - /* Exception containing an error message, if any. */ + /** + * Exception containing an error message, if any. + */ std::optional<Error> ex; Goal(Worker & worker, DerivedPath path) @@ -96,9 +124,11 @@ struct Goal : public std::enable_shared_from_this<Goal> return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 6bf60b297..d84f62877 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -8,16 +8,24 @@ namespace nix { struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e22180670..31fea9259 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional<TrustedFlag> isTrustedClient() override + { return NotTrusted; } }; @@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive); + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -2392,27 +2395,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { + auto rewriteRefs = [&]() -> StoreReferences { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair<bool, StorePathSet> res { - false, - {}, + StoreReferences res { + .self = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == *scratchPath) { - res.first = true; + res.self = true; } else if (auto outputRewrite = get(outputRewrites, origHash)) { std::string newRef = *outputRewrite; newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.others.insert(StorePath { newRef }); } else { - res.second.insert(r); + res.others.insert(r); } } return res; @@ -2445,18 +2447,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); - - auto finalPath = worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first); - if (*scratchPath != finalPath) { + ValidPathInfo newInfo0 { + worker.store, + outputPathName(drv->name, outputName), + FixedOutputInfo { + .hash = { + .method = outputHash.method, + .hash = got, + }, + .references = rewriteRefs(), + }, + Hash::dummy, + }; + if (*scratchPath != newInfo0.path) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); dumpPath(actualPath, rsink2); rsink2.flush(); }); @@ -2467,19 +2473,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } HashResult narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { - finalPath, - narHashAndSize.first, - }; - + newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); assert(newInfo0.ca); return newInfo0; @@ -2501,8 +2496,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) + newInfo0.references = std::move(refs.others); + if (refs.self) newInfo0.references.insert(newInfo0.path); return newInfo0; }, @@ -2516,7 +2511,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Check wanted hash */ const Hash & wanted = dof.hash.hash; assert(newInfo0.ca); - auto got = getContentAddressHash(*newInfo0.ca); + auto got = newInfo0.ca->getHash(); if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1c4b4e3fe..42d32a31a 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -10,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); - /* User selected for running the builder. */ + /** + * User selected for running the builder. + */ std::unique_ptr<UserLock> buildUser; - /* The process ID of the builder. */ + /** + * The process ID of the builder. + */ Pid pid; - /* The cgroup of the builder, if any. */ + /** + * The cgroup of the builder, if any. + */ std::optional<Path> cgroup; - /* The temporary directory. */ + /** + * The temporary directory. + */ Path tmpDir; - /* The path of the temporary directory in the sandbox. */ + /** + * The path of the temporary directory in the sandbox. + */ Path tmpDirInSandbox; - /* Master side of the pseudoterminal used for the builder's - standard output/error. */ + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ AutoCloseFD builderOut; - /* Pipe for synchronising updates to the builder namespaces. */ + /** + * Pipe for synchronising updates to the builder namespaces. + */ Pipe userNamespaceSync; - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxUserNamespace; - /* On Linux, whether we're doing the build in its own user - namespace. */ + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ bool usingUserNamespace = true; - /* Whether we're currently doing a chroot build. */ + /** + * Whether we're currently doing a chroot build. + */ bool useChroot = false; Path chrootRootDir; - /* RAII object to delete the chroot directory. */ + /** + * RAII object to delete the chroot directory. + */ std::shared_ptr<AutoDelete> autoDelChroot; - /* Whether to run the build in a private network namespace. */ + /** + * Whether to run the build in a private network namespace. + */ bool privateNetwork = false; - /* Stuff we need to pass to initChild(). */ + /** + * Stuff we need to pass to initChild(). + */ struct ChrootPath { Path source; bool optional; @@ -71,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ + /** + * Hash rewriting. + */ StringMap inputRewrites, outputRewrites; typedef map<StorePath, StorePath> RedirectedOutputs; RedirectedOutputs redirectedOutputs; - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. + /** + * The outputs paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. */ OutputPathMap scratchOutputs; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ + /** + * Path registration info from the previous round, if we're + * building multiple times. Since this contains the hash, it + * allows us to compare whether two rounds produced the same + * result. + */ std::map<Path, ValidPathInfo> prevInfos; uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } @@ -102,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; - /* The recursive Nix daemon socket. */ + /** + * The recursive Nix daemon socket. + */ AutoCloseFD daemonSocket; - /* The daemon main thread. */ + /** + * The daemon main thread. + */ std::thread daemonThread; - /* The daemon worker threads. */ + /** + * The daemon worker threads. + */ std::vector<std::thread> daemonWorkerThreads; - /* Paths that were added via recursive Nix calls. */ + /** + * Paths that were added via recursive Nix calls. + */ StorePathSet addedPaths; - /* Realisations that were added via recursive Nix calls. */ + /** + * Realisations that were added via recursive Nix calls. + */ std::set<DrvOutput> addedDrvOutputs; - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where - /nix/store/<bla> is some arbitrary path in a binary cache). */ + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where + * /nix/store/<bla> is some arbitrary path in a binary cache). + */ bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); @@ -138,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal virtual ~LocalDerivationGoal() override; - /* Whether we need to perform hash rewriting if there are valid output paths. */ + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ bool needsHashRewrite(); - /* The additional states. */ + /** + * The additional states. + */ void tryLocalBuild() override; - /* Start building a derivation. */ + /** + * Start building a derivation. + */ void startBuilder(); - /* Fill in the environment for the builder. */ + /** + * Fill in the environment for the builder. + */ void initEnv(); - /* Setup tmp dir location. */ + /** + * Setup tmp dir location. + */ void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ + /** + * Write a JSON file containing the derivation attributes. + */ void writeStructuredAttrs(); void startDaemon(); void stopDaemon(); - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ void addDependency(const StorePath & path); - /* Make a file owned by the builder. */ + /** + * Make a file owned by the builder. + */ void chownToBuilder(const Path & path); int getChildStatus() override; - /* Run the builder's process. */ + /** + * Run the builder's process. + */ void runChild(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ void closeReadPipes() override; - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ void cleanupHookFinally() override; void cleanupPreChildKill() override; void cleanupPostChildKill() override; @@ -196,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ + /** + * Delete the temporary directory, if we have one. + */ void deleteTmpDir(bool force); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ void killChild() override; - /* Kill any processes running under the build user UID or in the - cgroup of the build. */ + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ void killSandbox(bool getStats); - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ StorePath makeFallbackPath(std::string_view outputName); }; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 2af105b4d..190fb455a 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,7 +95,9 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1add9eb14..c2b7fc95a 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -11,38 +11,58 @@ class Worker; struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ + /** + * The path the substituter refers to the path as. This will be + * different when the stores have different names. + */ std::optional<StorePath> subPath; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list<ref<Store>> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr<Store> sub; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; - /* Path info returned by the substituter's query info operation. */ + /** + * Path info returned by the substituter's query info operation. + */ std::shared_ptr<const ValidPathInfo> info; - /* Pipe for the substituter's standard output. */ + /** + * Pipe for the substituter's standard output. + */ Pipe outPipe; - /* The substituter thread. */ + /** + * The substituter thread. + */ std::thread thr; std::promise<void> promise; - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ + /** + * Location where we're downloading the substitute. Differs from + * storePath when doing a repair. + */ Path destPath; std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, @@ -51,7 +71,9 @@ struct PathSubstitutionGoal : public Goal typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional<ContentAddress> ca; void done( @@ -65,16 +87,20 @@ public: void timedOut(Error && ex) override { abort(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; - /* The states. */ + /** + * The states. + */ void init(); void tryNext(); void gotInfo(); @@ -82,7 +108,9 @@ public: void tryToRun(); void finished(); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index d840b3b3f..48a1a27fa 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -17,24 +17,29 @@ struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal); typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and file descriptors for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; @@ -42,14 +47,19 @@ struct Child std::set<int> fds; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; /* Forward definition. */ struct HookInstance; -/* The worker class. */ +/** + * The worker class. + */ class Worker { private: @@ -57,38 +67,58 @@ private: /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list<Child> children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds and + * substitutions but not remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` where woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map<StorePath, bool> pathContentsGoodCache; public: @@ -97,17 +127,25 @@ public: const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; Store & store; @@ -129,16 +167,22 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ + /** + * Make a goal (with caching). + */ - /* derivation goal */ + /** + * derivation goal + */ private: std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -151,56 +195,80 @@ public: const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * substitution goal + */ std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build and substitution processes + * currently running (but not remote builds via the build + * hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ void childStarted(GoalPtr goal, const std::set<int> & fds, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); unsigned int exitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index cf32ccdc4..055b216db 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,7 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: @@ -21,39 +21,35 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) } } -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) -{ - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); -} - -std::string renderContentAddress(ContentAddress ca) +std::string ContentAddress::render() const { return std::visit(overloaded { - [](TextHash & th) { - return "text:" + th.hash.to_string(Base32, true); + [](const TextHash & th) { + return "text:" + + th.hash.to_string(Base32, true); }, - [](FixedOutputHash & fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + [](const FixedOutputHash & fsh) { + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } - }, ca); + }, raw); } -std::string renderContentAddressMethod(ContentAddressMethod cam) +std::string ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextHashMethod & th) { + [](const TextHashMethod & th) { return std::string{"text:"} + printHashType(htSHA256); }, - [](FixedOutputHashMethod & fshm) { + [](const FixedOutputHashMethod & fshm) { return "fixed:" + makeFileIngestionPrefix(fshm.fileIngestionMethod) + printHashType(fshm.hashType); } - }, cam); + }, raw); } -/* - Parses content address strings up to the hash. +/** + * Parses content address strings up to the hash. */ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & rest) { @@ -97,7 +93,7 @@ static ContentAddressMethod parseContentAddressMethodPrefix(std::string_view & r throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); } -ContentAddress parseContentAddress(std::string_view rawCa) { +ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; ContentAddressMethod caMethod = parseContentAddressMethodPrefix(rest); @@ -115,10 +111,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { .hash = Hash::parseNonSRIUnprefixed(rest, std::move(fohMethod.hashType)), }); }, - }, caMethod); + }, caMethod.raw); } -ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) +ContentAddressMethod ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -126,26 +122,55 @@ ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) return parseContentAddressMethodPrefix(asPrefixView); } -std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) +std::optional<ContentAddress> ContentAddress::parseOpt(std::string_view rawCaOpt) { - return rawCaOpt == "" ? std::optional<ContentAddress>() : parseContentAddress(rawCaOpt); + return rawCaOpt == "" + ? std::nullopt + : std::optional { ContentAddress::parse(rawCaOpt) }; }; std::string renderContentAddress(std::optional<ContentAddress> ca) { - return ca ? renderContentAddress(*ca) : ""; + return ca ? ca->render() : ""; } -Hash getContentAddressHash(const ContentAddress & ca) +const Hash & ContentAddress::getHash() const { return std::visit(overloaded { - [](const TextHash & th) { + [](const TextHash & th) -> auto & { return th.hash; }, - [](const FixedOutputHash & fsh) { + [](const FixedOutputHash & fsh) -> auto & { return fsh.hash; - } - }, ca); + }, + }, raw); +} + +bool StoreReferences::empty() const +{ + return !self && others.empty(); +} + +size_t StoreReferences::size() const +{ + return (self ? 1 : 0) + others.size(); +} + +ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](const TextHash & h) -> ContentAddressWithReferences { + return TextInfo { + .hash = h, + .references = {}, + }; + }, + [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + return FixedOutputInfo { + .hash = h, + .references = {}, + }; + }, + }, ca.raw); } } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 19fdfc1eb..2f98950fb 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,11 +3,30 @@ #include <variant> #include "hash.hh" +#include "path.hh" +#include "comparator.hh" namespace nix { +/* + * Content addressing method + */ + +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ + +/** + * The single way we can serialize "text" file system objects. + * + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + */ +struct TextHashMethod : std::monostate { }; + /** - * An enumeration of the ways we can serialize file system objects. + * An enumeration of the main ways we can serialize file system + * objects. */ enum struct FileIngestionMethod : uint8_t { /** @@ -22,6 +41,53 @@ enum struct FileIngestionMethod : uint8_t { }; /** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ +std::string makeFileIngestionPrefix(FileIngestionMethod m); + +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; + + GENERATE_CMP(FixedOutputHashMethod, me->fileIngestionMethod, me->hashType); +}; + +/** + * An enumeration of all the ways we can serialize file system objects. + * + * Just the type of a content address. Combine with the hash itself, and + * we have a `ContentAddress` as defined below. Combine that, in turn, + * with info on references, and we have `ContentAddressWithReferences`, + * as defined further below. + */ +struct ContentAddressMethod +{ + typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > Raw; + + Raw raw; + + GENERATE_CMP(ContentAddressMethod, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressMethod(auto &&... arg) + : raw(std::forward<decltype(arg)>(arg)...) + { } + + static ContentAddressMethod parse(std::string_view rawCaMethod); + + std::string render() const; +}; + + +/* + * Mini content address + */ + +/** * Somewhat obscure, used by \ref Derivation derivations and * `builtins.toFile` currently. */ @@ -30,10 +96,12 @@ struct TextHash { * Hash of the contents of the text/file. */ Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /** - * For path computed by makeFixedOutputPath. + * Used by most store objects that are content-addressed. */ struct FixedOutputHash { /** @@ -46,6 +114,8 @@ struct FixedOutputHash { Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /** @@ -60,41 +130,96 @@ struct FixedOutputHash { * - ‘fixed:<r?>:<ht>:<h>’: For paths computed by * Store::makeFixedOutputPath() / Store::addToStore(). */ -typedef std::variant< - TextHash, - FixedOutputHash -> ContentAddress; +struct ContentAddress +{ + typedef std::variant< + TextHash, + FixedOutputHash + > Raw; -/** - * Compute the prefix to the hash algorithm which indicates how the - * files were ingested. + Raw raw; + + GENERATE_CMP(ContentAddress, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddress(auto &&... arg) + : raw(std::forward<decltype(arg)>(arg)...) + { } + + /** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ + std::string render() const; + + static ContentAddress parse(std::string_view rawCa); + + static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt); + + const Hash & getHash() const; +}; + +std::string renderContentAddress(std::optional<ContentAddress> ca); + + +/* + * Full content address + * + * See the schema for store paths in store-api.cc */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); /** - * Compute the content-addressability assertion (ValidPathInfo::ca) for - * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + * A set of references to other store objects. + * + * References to other store objects are tracked with store paths, self + * references however are tracked with a boolean. */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); +struct StoreReferences { + /** + * References to other store objects + */ + StorePathSet others; -std::string renderContentAddress(ContentAddress ca); + /** + * Reference to this store object + */ + bool self = false; -std::string renderContentAddress(std::optional<ContentAddress> ca); + /** + * @return true iff no references, i.e. others is empty and self is + * false. + */ + bool empty() const; -ContentAddress parseContentAddress(std::string_view rawCa); + /** + * Returns the numbers of references, i.e. the size of others + 1 + * iff self is true. + */ + size_t size() const; -std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); + GENERATE_CMP(StoreReferences, me->self, me->others); +}; -Hash getContentAddressHash(const ContentAddress & ca); +// This matches the additional info that we need for makeTextPath +struct TextInfo { + TextHash hash; + /** + * References to other store objects only; self references + * disallowed + */ + StorePathSet references; -/* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + GENERATE_CMP(TextInfo, me->hash, me->references); +}; + +struct FixedOutputInfo { + FixedOutputHash hash; + /** + * References to other store objects or this one. + */ + StoreReferences references; + + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; /** @@ -102,13 +227,27 @@ struct FixedOutputHashMethod { * * A ContentAddress without a Hash. */ -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; +struct ContentAddressWithReferences +{ + typedef std::variant< + TextInfo, + FixedOutputInfo + > Raw; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + Raw raw; -std::string renderContentAddressMethod(ContentAddressMethod caMethod); + GENERATE_CMP(ContentAddressWithReferences, me->raw); + + /* The moral equivalent of `using Raw::Raw;` */ + ContentAddressWithReferences(auto &&... arg) + : raw(std::forward<decltype(arg)>(arg)...) + { } + + /** + * Create a ContentAddressWithReferences from a mere ContentAddress, by + * assuming no references in all cases. + */ + static ContentAddressWithReferences withoutRefs(const ContentAddress &); +}; } diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index a98f2a3b8..35216d470 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -12,8 +12,10 @@ struct Key std::string name; std::string key; - /* Construct Key from a string in the format - ‘<name>:<key-in-base64>’. */ + /** + * Construct Key from a string in the format + * ‘<name>:<key-in-base64>’. + */ Key(std::string_view s); std::string to_string() const; @@ -29,7 +31,9 @@ struct SecretKey : Key { SecretKey(std::string_view s); - /* Return a detached signature of the given string. */ + /** + * Return a detached signature of the given string. + */ std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -53,8 +57,10 @@ private: typedef std::map<std::string, PublicKey> PublicKeys; -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 656ad4587..ed2d15c10 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -401,21 +401,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - ContentAddressMethod contentAddressMethod = parseContentAddressMethod(camStr); + ContentAddressMethod contentAddressMethod = ContentAddressMethod::parse(camStr); FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { - [&](TextHashMethod &) { + [&](const TextHashMethod &) { // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, - [&](FixedOutputHashMethod & fohm) { + [&](const FixedOutputHashMethod & fohm) { auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, - }, contentAddressMethod); + }, contentAddressMethod.raw); }(); logger->stopWork(); @@ -880,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); - info.ca = parseContentAddressOpt(readString(from)); + info.ca = ContentAddress::parseOpt(readString(from)); from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; @@ -1032,6 +1032,15 @@ void processConnection( if (GET_PROTOCOL_MINOR(clientVersion) >= 33) to << nixVersion; + if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + // We and the underlying store both need to trust the client for + // it to be trusted. + auto temp = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }; + worker_proto::write(*store, to, temp); + } + /* Send startup error messages to the client. */ tunnelLogger->startWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67340a05b..1964c0d99 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -6,7 +6,6 @@ namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 06cc69056..0de36504b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,8 +36,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } @@ -313,6 +313,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an `std::string`. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector<char, 64 * 1024> buffer; @@ -889,6 +898,67 @@ std::optional<BasicDerivation> Derivation::tryResolve( return resolved; } + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional<DrvHash> hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + StorePath path = store.makeFixedOutputPath(drvName, { dof.hash, {} }); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( @@ -916,10 +986,79 @@ nlohmann::json DerivationOutput::toJSON( return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json) +{ + std::set<std::string_view> keys; + auto json = (std::map<std::string, nlohmann::json>) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> { + std::string hashAlgo = json["hashAlgo"]; + auto method = FileIngestionMethod::Flat; + if (hashAlgo.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = hashAlgo.substr(2); + } + auto hashType = parseHashType(hashAlgo); + return { method, hashType }; + }; + + if (keys == (std::set<std::string_view> { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .hash = { + .method = method, + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set<std::string_view> { "hashAlgo" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = method, + .hashType = hashType, + }; + } + + else if (keys == (std::set<std::string_view> { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = method, + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -950,4 +1089,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + Derivation res; + + res.name = json["name"]; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast<const StringSet &>(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index e12bd2119..ccdde36ca 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -7,6 +7,7 @@ #include "content-address.hh" #include "repair-flag.hh" #include "sync.hh" +#include "comparator.hh" #include <map> #include <variant> @@ -24,6 +25,8 @@ class Store; struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; /** @@ -44,6 +47,8 @@ struct DerivationOutputCAFixed * @param outputName The name of this output. */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->hash); }; /** @@ -62,13 +67,17 @@ struct DerivationOutputCAFloating * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; /** * Input-addressed output which depends on a (CA) derivation whose hash * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; /** * Impure output which is moved to a content-addressed location (like @@ -85,6 +94,8 @@ struct DerivationOutputImpure * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -125,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json); }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; @@ -242,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw { struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -273,6 +295,15 @@ struct BasicDerivation DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation @@ -308,11 +339,26 @@ struct Derivation : BasicDerivation Store & store, const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; + /* Check that the derivation is valid and does not present any + illegal states. + + This is mainly a matter of checking the outputs, where our C++ + representation supports all sorts of combinations we do not yet + allow. */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast<const BasicDerivation &>(*me), + me->inputDrvs); }; @@ -389,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * * A fixed-output derivation is a derivation whose outputs have a * specified content hash and hash algorithm. (Currently they must have - * exactly one output (`out'), which is specified using the `outputHash' - * and `outputHashAlgo' attributes, but the algorithm doesn't assume + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume * this.) We don't want changes to such derivations to propagate upwards * through the dependency graph, changing output paths everywhere. * - * For instance, if we change the url in a call to the `fetchurl' + * For instance, if we change the url in a call to the `fetchurl` * function, we do not want to rebuild everything depending on it---after * all, (the hash of) the file being downloaded is unchanged. So the * *output paths* should not change. On the other hand, the *derivation diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e5f0f1b33..9a2ffda39 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) - + "!" + + '^' + + outputs.to_string(); +} + +std::string DerivedPath::Built::to_string_legacy(const Store & store) const +{ + return store.printStorePath(drvPath) + + '!' + outputs.to_string(); } std::string DerivedPath::to_string(const Store & store) const { - return std::visit( - [&](const auto & req) { return req.to_string(store); }, - this->raw()); + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const Store & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } @@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi }; } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) { - size_t n = s.find("!"); + size_t n = s.find(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + return parseWith(store, s, "^"); +} + +DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +{ + return parseWith(store, s, "!"); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 2155776b1..5f7acbebc 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -48,8 +48,18 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + */ + static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); nlohmann::json toJSON(ref<Store> store) const; GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); @@ -81,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw { return static_cast<const Raw &>(*this); } + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + */ static DerivedPath parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + static DerivedPath parseLegacy(const Store & store, std::string_view); }; /** diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 16e5fafd7..74d6ed3b5 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -39,6 +39,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store callback(nullptr); } + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional<TrustedFlag> isTrustedClient() override + { + return Trusted; + } + static std::set<std::string> uriSchemes() { return {"dummy"}; } @@ -63,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void queryRealisationUncached(const DrvOutput &, Callback<std::shared_ptr<const Realisation>> callback) noexcept override { callback(nullptr); } + + virtual ref<FSAccessor> getFSAccessor() override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 1c98a42d7..1df19e647 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -5,8 +5,10 @@ namespace nix { -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ +/** + * An abstract class for accessing a filesystem-like structure, such + * as a (possibly remote) Nix store or the contents of a NAR file. + */ class FSAccessor { public: @@ -15,8 +17,17 @@ public: struct Stat { Type type = tMissing; - uint64_t fileSize = 0; // regular files only + /** + * regular files only + */ + uint64_t fileSize = 0; + /** + * regular files only + */ bool isExecutable = false; // regular files only + /** + * regular files only + */ uint64_t narOffset = 0; // regular files only }; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 17f043a63..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -12,19 +12,20 @@ typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -34,28 +35,38 @@ struct GCOptions GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; @@ -64,21 +75,27 @@ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ + /** + * Add an indirect root, which is merely a symlink to `path` from + * `/nix/var/nix/gcroots/auto/<hash of path>`. `path` is supposed + * to be a symlink to a store path. The garbage collector will + * automatically remove the indirect root when it finds that + * `path` has disappeared. + */ virtual void addIndirectRoot(const Path & path) = 0; - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 823b4af74..1b38e32fb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,12 +7,20 @@ #include <algorithm> #include <map> +#include <mutex> #include <thread> #include <dlfcn.h> #include <sys/utsname.h> #include <nlohmann/json.hpp> +#include <sodium/core.h> + +#ifdef __GLIBC__ +#include <gnu/lib-names.h> +#include <nss.h> +#include <dlfcn.h> +#endif namespace nix { @@ -41,7 +49,6 @@ Settings::Settings() , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); @@ -281,6 +288,42 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -291,6 +334,24 @@ void assertLibStoreInitialized() { } void initLibStore() { + + initLibUtil(); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + + loadConfFile(); + + preloadNSS(); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + initLibStoreDone = true; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9b04c9e78..d6c5d437a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -72,30 +72,46 @@ public: Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector<Path> nixUserConfFiles; - /* The directory where the main programs are stored. */ + /** + * The directory where the main programs are stored. + */ Path nixBinDir; - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", @@ -121,7 +137,9 @@ public: )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; Setting<size_t> logLines{this, 10, "log-lines", @@ -157,8 +175,10 @@ public: )", {"build-cores"}, false}; - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting<std::string> thisSystem{ @@ -308,16 +328,6 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` )"}; Setting<uint32_t> startId{this, @@ -347,16 +357,6 @@ public: Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` )"}; #endif @@ -458,9 +458,6 @@ public: )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - Setting<SandboxMode> sandboxMode{ this, #if __linux__ @@ -997,8 +994,10 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ +/** + * This should be called after settings are initialized, but before + * anything else + */ void initPlugins(); void loadConfFile(); @@ -1008,12 +1007,16 @@ std::vector<Path> getUserConfigFiles(); extern const std::string nixVersion; -/* NB: This is not sufficient. You need to call initNix() */ +/** + * NB: This is not sufficient. You need to call initNix() + */ void initLibStore(); -/* It's important to initialize before doing _anything_, which is why we - call upon the programmer to handle this correctly. However, we only add - this in a key locations, so as not to litter the code. */ +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ void assertLibStoreInitialized(); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 238fd1d98..85c5eed4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,18 @@ protected: }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional<TrustedFlag> isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 98322b045..c3cb3032a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -156,7 +156,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor throw Error("NAR hash is now mandatory"); info->narHash = Hash::parseAnyPrefixed(s); } - info->ca = parseContentAddressOpt(readString(conn->from)); + info->ca = ContentAddress::parseOpt(readString(conn->from)); info->sigs = readStrings<StringSet>(conn->from); auto s = readString(conn->from); @@ -342,6 +342,9 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } + virtual ref<FSAccessor> getFSAccessor() override + { unsupported("getFSAccessor"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -389,6 +392,15 @@ public: return conn->remoteVersion; } + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional<TrustedFlag> isTrustedClient() override + { + return std::nullopt; + } + void queryRealisationUncached(const DrvOutput &, Callback<std::shared_ptr<const Realisation>> callback) noexcept override // TODO: Implement diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index e5ee6fc15..5481dd762 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -95,6 +95,10 @@ protected: return paths; } + std::optional<TrustedFlag> isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 1e7f31a09..a03bb88f5 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -48,7 +48,9 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; - /* Register a permanent GC root. */ + /** + * Register a permanent GC root. + */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f58d90895..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -711,61 +711,6 @@ void canonicalisePathMetaData(const Path & path, } -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional<DrvHash> hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -876,7 +821,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -944,7 +889,7 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s if (s) info->sigs = tokenizeString<StringSet>(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = ContentAddress::parseOpt(s); /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -1175,8 +1120,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an @@ -1278,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if<FixedOutputHash>(&*info.ca)) { + if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) { auto actualFoHash = hashCAPath( foHash->method, foHash->hash.type, @@ -1291,7 +1235,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, actualFoHash.hash.to_string(Base32, true)); } } - if (auto textHash = std::get_if<TextHash>(&*info.ca)) { + if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) { auto actualTextHash = hashString(htSHA256, readFile(realPath)); if (textHash->hash != actualTextHash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1377,7 +1321,19 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name, references); + ContentAddressWithReferences desc = FixedOutputInfo { + .hash = { + .method = method, + .hash = hash, + }, + .references = { + .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus + .self = false, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(name, desc); addTempRoot(dstPath); @@ -1397,7 +1353,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1421,10 +1377,13 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { + *this, + name, + std::move(desc), + narHash.first + }; info.narSize = narHash.second; - info.references = references; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1441,7 +1400,10 @@ StorePath LocalStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); @@ -1685,6 +1647,11 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } +std::optional<TrustedFlag> LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1b668b6fd..55add18dd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -19,10 +19,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -51,30 +55,40 @@ class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr<Stmts> stmts; - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future<void> gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); std::unique_ptr<PublicKeys> publicKeys; @@ -97,11 +111,15 @@ private: public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); LocalStore(std::string scheme, std::string path, const Params & params); @@ -110,7 +128,9 @@ public: static std::set<std::string> uriSchemes() { return {}; } - /* Implementations of abstract store API methods. */ + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -155,13 +175,19 @@ private: void createTempRootsFile(); - /* The file to which we write our temporary roots. */ + /** + * The file to which we write our temporary roots. + */ Sync<AutoCloseFD> _fdTempRoots; - /* The global GC lock. */ + /** + * The global GC lock. + */ Sync<AutoCloseFD> _fdGCLock; - /* Connection to the garbage collector. */ + /** + * Connection to the garbage collector. + */ Sync<AutoCloseFD> _fdRootsSocket; public: @@ -180,42 +206,54 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; + std::optional<TrustedFlag> isTrustedClient() override; + void vacuumDB(); void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -245,7 +283,9 @@ private: void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); void verifyPath(const Path & path, const StringSet & store, @@ -268,8 +308,6 @@ private: std::pair<Path, AutoCloseFD> createTempDirInStore(); - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); - typedef std::unordered_set<ino_t> InodeHash; InodeHash loadInodeHash(); @@ -280,8 +318,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); @@ -311,18 +351,23 @@ typedef std::pair<dev_t, ino_t> Inode; typedef std::set<Inode> InodesSeen; -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. - If uidRange is not empty, this function will throw an error if it - encounters files owned by a user outside of the closed interval - [uidRange->first, uidRange->second]. -*/ +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ void canonicalisePathMetaData( const Path & path, std::optional<std::pair<uid_t, uid_t>> uidRange, diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 7be3ba314..1c268e1fb 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -13,14 +13,18 @@ struct UserLock { virtual ~UserLock() { } - /* Get the first and last UID. */ + /** + * Get the first and last UID. + */ std::pair<uid_t, uid_t> getUIDRange() { auto first = getUID(); return {first, first + getUIDCount() - 1}; } - /* Get the first UID. */ + /** + * Get the first UID. + */ virtual uid_t getUID() = 0; virtual uid_t getUIDCount() = 0; @@ -30,8 +34,10 @@ struct UserLock virtual std::vector<gid_t> getSupplementaryGIDs() = 0; }; -/* Acquire a user lock for a UID range of size `nrIds`. Note that this - may return nullptr if no user is available. */ +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index 7aeec73b2..a84f7dbeb 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -10,8 +10,10 @@ struct LogStore : public virtual Store { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ std::optional<std::string> getBuildLog(const StorePath & path); virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0; diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 64d172918..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,18 +27,17 @@ std::map<StorePath, StorePath> makeContentAddressed( StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + StoreReferences refs; for (auto & ref : oldInfo->references) { if (ref == path) - hasSelfReference = true; + refs.self = true; else { auto i = remappings.find(ref); auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - references.insert(std::move(replacement)); + refs.others.insert(std::move(replacement)); } } @@ -49,24 +48,28 @@ std::map<StorePath, StorePath> makeContentAddressed( auto narModuloHash = hashModuloSink.finish().first; - auto dstPath = dstStore.makeFixedOutputPath( - FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + ValidPathInfo info { + dstStore, + path.name(), + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }, + .references = std::move(refs), + }, + Hash::dummy, + }; - printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path)); StringSink sink2; - RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2); rsink2(sink.s); rsink2.flush(); - ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.narHash = hashString(htSHA256, sink2.s); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }; StringSource source(sink2.s); dstStore.addToStore(info, source); diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 940e537b6..5e19bd3c7 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -10,24 +10,30 @@ namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ +/** + * Return an object that provides access to the contents of a NAR + * file. + */ ref<FSAccessor> makeNarAccessor(std::string && nar); ref<FSAccessor> makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; ref<FSAccessor> makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -/* Write a JSON representation of the contents of a NAR (except file - contents). */ +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 2645f468b..c7176d30f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -273,7 +273,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); + narInfo->ca = ContentAddress::parseOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index c596f2d71..bbd1d05d5 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -43,8 +43,10 @@ public: const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref<NarInfoDiskCache> getNarInfoDiskCache(); ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 071d8355e..274cd861c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -74,7 +74,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "CA") { if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? - ca = parseContentAddressOpt(value); + ca = ContentAddress::parseOpt(value); } pos = eol + 1; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 3cae8e659..5dbdafac3 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; + NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index bd55a9d06..e60d7abe0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,25 +21,45 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } - -bool ValidPathInfo::isContentAddressed(const Store & store) const +std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const { - if (! ca) return false; - - auto caPath = std::visit(overloaded { - [&](const TextHash & th) { - return store.makeTextPath(path.name(), th.hash, references); + if (! ca) + return std::nullopt; + + return std::visit(overloaded { + [&](const TextHash & th) -> ContentAddressWithReferences { + assert(references.count(path) == 0); + return TextInfo { + .hash = th, + .references = references, + }; }, - [&](const FixedOutputHash & fsh) { + [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); + return FixedOutputInfo { + .hash = foh, + .references = { + .others = std::move(refs), + .self = hasSelfReference, + }, + }; + }, + }, ca->raw); +} + +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + auto fullCaOpt = contentAddressWithReferences(); + + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(path.name(), *fullCaOpt); bool res = caPath == path; @@ -77,6 +97,29 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + std::string_view name, + ContentAddressWithReferences && ca, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(name, ca)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo && ti) { + this->references = std::move(ti.references); + this->ca = std::move((TextHash &&) ti); + }, + [this](FixedOutputInfo && foi) { + this->references = std::move(foi.references.others); + if (foi.references.self) + this->references.insert(path); + this->ca = std::move((FixedOutputHash &&) foi); + }, + }, std::move(ca).raw); +} + + ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) { return read(source, store, format, store.parseStorePath(readString(source))); @@ -93,7 +136,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned if (format >= 16) { source >> info.ultimate; info.sigs = readStrings<StringSet>(source); - info.ca = parseContentAddressOpt(readString(source)); + info.ca = ContentAddress::parseOpt(readString(source)); } return info; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b28bf751c..221523622 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -19,8 +19,14 @@ struct SubstitutablePathInfo { std::optional<StorePath> deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; @@ -30,35 +36,40 @@ struct ValidPathInfo { StorePath path; std::optional<StorePath> deriver; - // TODO document this + /** + * \todo document this + */ Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional<ContentAddress> ca; bool operator == (const ValidPathInfo & i) const @@ -69,27 +80,42 @@ struct ValidPathInfo && references == i.references; } - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; void sign(const Store & store, const SecretKey & secretKey); - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ + std::optional<ContentAddressWithReferences> contentAddressWithReferences() const; + + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; static const size_t maxSigs = std::numeric_limits<size_t>::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; @@ -99,6 +125,9 @@ struct ValidPathInfo ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + virtual ~ValidPathInfo() { } static ValidPathInfo read(Source & source, const Store & store, unsigned int format); diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index a845b0e5f..d75850868 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -6,13 +6,14 @@ namespace nix { -/* This is a deprecated old type just for use by the old CLI, and older - versions of the RPC protocols. In new code don't use it; you want - `DerivedPath` instead. - - `DerivedPath` is better because it handles more cases, and does so more - explicitly without devious punning tricks. -*/ +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; @@ -31,9 +32,11 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s); class Store; -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index ba8f2749b..4921df352 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -5,12 +5,16 @@ namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ +/** + * Delete an open lock file. + */ void deleteLockFile(const Path & path, int fd); enum LockType { ltRead, ltWrite, ltNone }; diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 6a5965390..4e1f42e83 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" + #include "types.hh" #include "pathlocks.hh" #include <time.h> @@ -24,9 +24,11 @@ struct Generation typedef std::list<Generation> Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Returns the list of currently present generations for the specified + * profile, sorted by generation number. Also returns the number of + * the current generation. + */ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); class LocalFSStore; @@ -47,26 +49,32 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional<GenerationNumber> dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); /** diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d24d83117..d0ce0bce9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,9 +42,43 @@ void write(const Store & store, Sink & out, const StorePath & storePath) } +std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _) +{ + auto temp = readNum<uint8_t>(from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted) +{ + if (!optTrusted) + out << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + out << (uint8_t)1; + break; + case NotTrusted: + out << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) { - return parseContentAddress(readString(from)); + return ContentAddress::parse(readString(from)); } void write(const Store & store, Sink & out, const ContentAddress & ca) @@ -56,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _) { auto s = readString(from); - return DerivedPath::parse(store, s); + return DerivedPath::parseLegacy(store, s); } void write(const Store & store, Sink & out, const DerivedPath & req) { - out << req.to_string(store); + out << req.to_string_legacy(store); } @@ -134,7 +168,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) { - return parseContentAddressOpt(readString(from)); + return ContentAddress::parseOpt(readString(from)); } void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) @@ -226,6 +260,13 @@ void RemoteStore::initConnection(Connection & conn) conn.daemonNixVersion = readString(conn.from); } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { + conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {}); + } else { + // We don't know the answer; protocol to old. + conn.remoteTrustsUs = std::nullopt; + } + auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); } @@ -545,7 +586,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( conn->to << wopAddToStore << name - << renderContentAddressMethod(caMethod); + << caMethod.render(); worker_proto::write(*this, conn->to, references); conn->to << repair; @@ -603,7 +644,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( } } - }, caMethod); + }, caMethod.raw); auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -1082,6 +1123,11 @@ unsigned int RemoteStore::getProtocol() return conn->daemonVersion; } +std::optional<TrustedFlag> RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f5f45f853..1c45f543e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -32,8 +32,10 @@ struct RemoteStoreConfig : virtual StoreConfig "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -69,7 +71,9 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref<const ValidPathInfo> addCAToStore( Source & dump, std::string_view name, @@ -77,7 +81,9 @@ public: const StorePathSet & references, RepairFlag repair); - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + /** + * Add a content-addressable store path. Does not support references. `dump` will be drained. + */ StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; @@ -144,6 +150,8 @@ public: unsigned int getProtocol() override; + std::optional<TrustedFlag> isTrustedClient() override; + void flushBadConnections(); struct Connection @@ -151,6 +159,7 @@ public: FdSink to; FdSource from; unsigned int daemonVersion; + std::optional<TrustedFlag> remoteTrustsUs; std::optional<std::string> daemonNixVersion; std::chrono::time_point<std::chrono::steady_clock> startTime; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ac82147ee..d2fc6abaf 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -509,6 +509,16 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return paths; } + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional<TrustedFlag> isTrustedClient() override + { + return std::nullopt; + } + static std::set<std::string> uriSchemes() { return {"s3"}; } }; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 871f2f3be..df334c23c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,14 +239,11 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - static std::atomic<time_t> lastWarned{0}; - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 86410f998..6e14852cb 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -11,7 +11,9 @@ struct sqlite3_stmt; namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; @@ -23,7 +25,9 @@ struct SQLite ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -31,7 +35,9 @@ struct SQLite uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -43,7 +49,9 @@ struct SQLiteStmt ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -56,7 +64,9 @@ struct SQLiteStmt ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -64,11 +74,15 @@ struct SQLiteStmt int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -82,8 +96,10 @@ struct SQLiteStmt } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -123,18 +139,22 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template<typename T, typename F> T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + handleSQLiteBusy(e, nextWarning); } } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8fc3ed46a..5bee1af9f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "nar-info-disk-cache.hh" #include "thread-pool.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" @@ -98,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). <id> is the name of the output (usually, "out"). - <h2> = base-16 representation of a SHA-256 hash of: + <h2> = base-16 representation of a SHA-256 hash of <s2> + + <s2> = if <type> = "text:...": the string written to the resulting store path - if <type> = "source": + if <type> = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if <type> = "output:<id>": @@ -162,63 +165,63 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in, say, <s2> (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const StoreReferences & references) { - for (auto & i : references) { + for (auto & i : references.others) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.self) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { - assert(references.empty()); + assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", StoreReferences { + .others = info.references, + .self = false, + }), + info.hash.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { - [&](const TextHash & th) { - return makeTextPath(name, th.hash, references); + [&](const TextInfo & ti) { + return makeTextPath(name, ti); }, - [&](const FixedOutputHash & fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](const FixedOutputInfo & foi) { + return makeFixedOutputPath(name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, ca.raw); } @@ -228,7 +231,14 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + .hash = { + .method = method, + .hash = h, + }, + .references = {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -237,7 +247,10 @@ StorePath Store::computeStorePathForText( std::string_view s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -425,11 +438,18 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + name, + FixedOutputInfo { + .hash = { + .method = method, + .hash = hash, + }, + .references = {}, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -521,7 +541,9 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA( + path.first.name(), + ContentAddressWithReferences::withoutRefs(*path.second)); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -538,10 +560,11 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, - info->references, - narInfo ? narInfo->fileSize : 0, - info->narSize}); + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -1025,7 +1048,9 @@ void copyStorePath( // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore.makeFixedOutputPathFromCA( + info->path.name(), + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1137,7 +1162,9 @@ std::map<StorePath, StorePath> copyPaths( auto storePathForSrc = currentPathInfo.path; auto storePathForDst = storePathForSrc; if (currentPathInfo.ca && currentPathInfo.references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + storePathForDst = dstStore.makeFixedOutputPathFromCA( + currentPathInfo.path.name(), + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d1047380..74f50a00d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -89,6 +89,7 @@ const uint32_t exportMagic = 0x4558494e; enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; @@ -267,17 +268,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** * Preparatory part of addToStore(). @@ -410,17 +405,17 @@ public: { unsupported("queryReferrers"); } /** - * @return all currently valid derivations that have `path' as an + * @return all currently valid derivations that have `path` as an * output. * - * (Note that the result of `queryDeriver()' is the derivation that - * was actually used to produce `path', which may not exist + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist * anymore.) */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /** - * Query the outputs of the derivation denoted by `path'. + * Query the outputs of the derivation denoted by `path`. */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); @@ -512,7 +507,7 @@ public: /** * Like addToStore(), but the contents of the path are contained - * in `dump', which is either a NAR serialisation (if recursive == + * in `dump`, which is either a NAR serialisation (if recursive == * true) or simply the contents of a regular file (if recursive == * false). * `dump` may be drained @@ -633,8 +628,8 @@ public: /** * @return a string representing information about the path that - * can be loaded into the database using `nix-store --load-db' or - * `nix-store --register-validity'. + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); @@ -677,8 +672,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref<FSAccessor> getFSAccessor() - { unsupported("getFSAccessor"); } + virtual ref<FSAccessor> getFSAccessor() = 0; /** * Repair the contents of the given path by redownloading it using @@ -714,12 +708,12 @@ public: /** * @param [out] out Place in here the set of all store paths in the - * file system closure of `storePath'; that is, all paths than can - * be directly or indirectly reached from it. `out' is not cleared. + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. * * @param flipDirection If true, the set of paths that can reach - * `storePath' is returned; that is, the closures under the - * `referrers' relation instead of the `references' relation is + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is * returned. */ virtual void computeFSClosure(const StorePathSet & paths, @@ -815,6 +809,17 @@ public: return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional<TrustedFlag> isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 12be8504d..80ee52fd0 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest { }; -#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ - TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ +#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json)); \ } -TEST_JSON(DerivationOutput, inputAddressed, +TEST_JSON(inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixed, +TEST_JSON(caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -42,7 +56,7 @@ TEST_JSON(DerivationOutput, caFixed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFloating, +TEST_JSON(caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -52,12 +66,12 @@ TEST_JSON(DerivationOutput, caFloating, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, deferred, +TEST_JSON(deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(DerivationOutput, impure, +TEST_JSON(impure, R"({ "hashAlgo": "r:sha256", "impure": true @@ -68,8 +82,28 @@ TEST_JSON(DerivationOutput, impure, }), "drv-name", "output-name") -TEST_JSON(Derivation, impure, +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ + } + +TEST_JSON(simple, R"({ + "name": "my-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -92,6 +126,7 @@ TEST_JSON(Derivation, impure, })", ({ Derivation drv; + drv.name = "my-derivation"; drv.inputSrcs = { store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; @@ -117,7 +152,8 @@ TEST_JSON(Derivation, impure, }, }; drv; - })) + }), + "drv-name") #undef TEST_JSON diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d1ac2c5e7..e6d32dbd0 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -53,6 +53,14 @@ TEST_F(DerivedPathTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, prop_round_rip, (const DerivedPath & o)) { diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 697dd2b8c..c7a6f8688 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -10,7 +10,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +#define PROTOCOL_VERSION (1 << 8 | 35) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -103,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, std::optional<TrustedFlag>); MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); MAKE_WORKER_PROTO(template<typename T>, std::set<T>); diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh index 54721649c..86becafa6 100644 --- a/src/libutil/ansicolor.hh +++ b/src/libutil/ansicolor.hh @@ -1,9 +1,12 @@ #pragma once -///@file +/** + * @file + * + * @brief Some ANSI escape sequences. + */ namespace nix { -/* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" #define ANSI_FAINT "\e[2m" diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 60e33dd40..2cf164a41 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -20,39 +20,41 @@ namespace nix { * * The format is as follows: * - * IF path points to a REGULAR FILE: - * dump(path) = attrs( - * [ ("type", "regular") - * , ("contents", contents(path)) - * ]) + * ``` + * IF path points to a REGULAR FILE: + * dump(path) = attrs( + * [ ("type", "regular") + * , ("contents", contents(path)) + * ]) * - * IF path points to a DIRECTORY: - * dump(path) = attrs( - * [ ("type", "directory") - * , ("entries", concat(map(f, sort(entries(path))))) - * ]) - * where f(fn) = attrs( - * [ ("name", fn) - * , ("file", dump(path + "/" + fn)) - * ]) + * IF path points to a DIRECTORY: + * dump(path) = attrs( + * [ ("type", "directory") + * , ("entries", concat(map(f, sort(entries(path))))) + * ]) + * where f(fn) = attrs( + * [ ("name", fn) + * , ("file", dump(path + "/" + fn)) + * ]) * - * where: + * where: * - * attrs(as) = concat(map(attr, as)) + encN(0) - * attrs((a, b)) = encS(a) + encS(b) + * attrs(as) = concat(map(attr, as)) + encN(0) + * attrs((a, b)) = encS(a) + encS(b) * - * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + * encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) * - * encN(n) = 64-bit little-endian encoding of n. + * encN(n) = 64-bit little-endian encoding of n. * - * contents(path) = the contents of a regular file. + * contents(path) = the contents of a regular file. * - * sort(strings) = lexicographic sort by 8-bit value (strcmp). + * sort(strings) = lexicographic sort by 8-bit value (strcmp). * - * entries(path) = the entries of a directory, without `.' and - * `..'. + * entries(path) = the entries of a directory, without `.` and + * `..`. * - * `+' denotes string concatenation. + * `+` denotes string concatenation. + * ``` */ void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); @@ -88,7 +90,7 @@ struct ParseSink /** * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s'. Otherwise barf. + * the contents of the file to `s`. Otherwise barf. */ struct RetrieveRegularNARSink : ParseSink { diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 76e48c4f2..614883c06 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -192,8 +192,10 @@ public: */ bool isAllowed(const std::set<CanonPath> & allowed) const; - /* Return a representation `x` of `path` relative to `this`, i.e. - `CanonPath(this.makeRelative(x), this) == path`. */ + /** + * Return a representation `x` of `path` relative to `this`, i.e. + * `CanonPath(this.makeRelative(x), this) == path`. + */ std::string makeRelative(const CanonPath & path) const; }; diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 2b5424b3d..9f661c5c3 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -17,12 +17,12 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ + __VA_OPT__(const MY_TYPE* me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 89180e7a7..ba0847cde 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -23,7 +23,7 @@ struct ChunkedCompressionSink : CompressionSink { uint8_t outbuf[32 * 1024]; - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { const size_t CHUNK_SIZE = sizeof(outbuf) << 2; while (!data.empty()) { @@ -103,7 +103,7 @@ struct ArchiveCompressionSink : CompressionSink throw Error(reason, archive_error_string(this->archive)); } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); if (result <= 0) check(result); @@ -136,7 +136,7 @@ struct NoneSink : CompressionSink warn("requested compression level '%d' not supported by compression method 'none'", level); } void finish() override { flush(); } - void write(std::string_view data) override { nextSink(data); } + void writeUnbuffered(std::string_view data) override { nextSink(data); } }; struct BrotliDecompressionSink : ChunkedCompressionSink diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 3892831c2..4e53a7b3c 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -12,7 +12,7 @@ namespace nix { struct CompressionSink : BufferedSink, FinishSink { using BufferedSink::operator (); - using BufferedSink::write; + using BufferedSink::writeUnbuffered; using FinishSink::finish; }; diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8d63536d6..a42f3a849 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -70,17 +70,10 @@ void AbstractConfig::reapplyUnknownSettings() set(s.first, s.second); } -// Whether we should process the option. Excludes aliases, which are handled elsewhere, and disabled features. -static bool applicable(const Config::SettingData & sd) -{ - return !sd.isAlias - && experimentalFeatureSettings.isEnabled(sd.setting->experimentalFeature); -} - void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly) { for (auto & opt : _settings) - if (applicable(opt.second) && (!overriddenOnly || opt.second.setting->overridden)) + if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -154,7 +147,7 @@ nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); for (auto & s : _settings) - if (applicable(s.second)) + if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; } @@ -163,7 +156,7 @@ std::string Config::toKeyValue() { auto res = std::string(); for (auto & s : _settings) - if (applicable(s.second)) + if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; } @@ -171,9 +164,6 @@ std::string Config::toKeyValue() void Config::convertToArgs(Args & args, const std::string & category) { for (auto & s : _settings) { - /* We do include args for settings gated on disabled - experimental-features. The args themselves however will also be - gated on any experimental feature the underlying setting is. */ if (!s.second.isAlias) s.second.setting->convertToArg(args, category); } @@ -201,6 +191,10 @@ std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject() std::map<std::string, nlohmann::json> obj; obj.emplace("description", description); obj.emplace("aliases", aliases); + if (experimentalFeature) + obj.emplace("experimentalFeature", *experimentalFeature); + else + obj.emplace("experimentalFeature", nullptr); return obj; } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 3c1d70294..162626791 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -371,8 +371,23 @@ extern GlobalConfig globalConfig; struct ExperimentalFeatureSettings : Config { - Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; + Setting<std::set<ExperimentalFeature>> experimentalFeatures{ + this, {}, "experimental-features", + R"( + Experimental features that are enabled. + + Example: + + ``` + experimental-features = nix-command flakes + ``` + + The following experimental features are available: + + {{#include experimental-features-shortlist.md}} + + Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + )"}; /** * Check whether the given experimental feature is enabled. diff --git a/src/libutil/error.hh b/src/libutil/error.hh index eafc6a540..6a0923081 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,5 +1,19 @@ #pragma once -///@file +/** + * @file + * + * @brief This file defines two main structs/classes used in nix error handling. + * + * ErrorInfo provides a standard payload of error information, with conversion to string + * happening in the logger rather than at the call site. + * + * BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains + * an ErrorInfo. + * + * ErrorInfo structs are sent to the logger as part of an exception, or directly with the + * logError or logWarning macros. + * See libutil/tests/logging.cc for usage examples. + */ #include "suggestions.hh" #include "ref.hh" @@ -27,22 +41,6 @@ namespace nix { -/* - - This file defines two main structs/classes used in nix error handling. - - ErrorInfo provides a standard payload of error information, with conversion to string - happening in the logger rather than at the call site. - - BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains - an ErrorInfo. - - ErrorInfo structs are sent to the logger as part of an exception, or directly with the - logError or logWarning macros. - - See libutil/tests/logging.cc for usage examples. - - */ typedef enum { lvlError = 0, diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 58d762ebb..5b4418714 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -5,29 +5,209 @@ namespace nix { -std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { - { Xp::CaDerivations, "ca-derivations" }, - { Xp::ImpureDerivations, "impure-derivations" }, - { Xp::Flakes, "flakes" }, - { Xp::NixCommand, "nix-command" }, - { Xp::RecursiveNix, "recursive-nix" }, - { Xp::NoUrlLiterals, "no-url-literals" }, - { Xp::FetchClosure, "fetch-closure" }, - { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::Cgroups, "cgroups" }, - { Xp::DiscardReferences, "discard-references" }, +struct ExperimentalFeatureDetails +{ + ExperimentalFeature tag; + std::string_view name; + std::string_view description; }; +constexpr std::array<ExperimentalFeatureDetails, 11> xpFeatureDetails = {{ + { + .tag = Xp::CaDerivations, + .name = "ca-derivations", + .description = R"( + Allow derivations to be content-addressed in order to prevent + rebuilds when changes to the derivation do not result in changes to + the derivation's output. See + [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) + for details. + )", + }, + { + .tag = Xp::ImpureDerivations, + .name = "impure-derivations", + .description = R"( + Allow derivations to produce non-fixed outputs by setting the + `__impure` derivation attribute to `true`. An impure derivation can + have differing outputs each time it is built. + + Example: + + ``` + derivation { + name = "impure"; + builder = /bin/sh; + __impure = true; # mark this derivation as impure + args = [ "-c" "read -n 10 random < /dev/random; echo $random > $out" ]; + system = builtins.currentSystem; + } + ``` + + Each time this derivation is built, it can produce a different + output (as the builder outputs random bytes to `$out`). Impure + derivations also have access to the network, and only fixed-output + or other impure derivations can rely on impure derivations. Finally, + an impure derivation cannot also be + [content-addressed](#xp-feature-ca-derivations). + )", + }, + { + .tag = Xp::Flakes, + .name = "flakes", + .description = R"( + Enable flakes. See the manual entry for [`nix + flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. + )", + }, + { + .tag = Xp::NixCommand, + .name = "nix-command", + .description = R"( + Enable the new `nix` subcommands. See the manual on + [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. + )", + }, + { + .tag = Xp::RecursiveNix, + .name = "recursive-nix", + .description = R"( + Allow derivation builders to call Nix, and thus build derivations + recursively. + + Example: + + ``` + with import <nixpkgs> {}; + + runCommand "foo" + { + buildInputs = [ nix jq ]; + NIX_PATH = "nixpkgs=${<nixpkgs>}"; + } + '' + hello=$(nix-build -E '(import <nixpkgs> {}).hello.overrideDerivation (args: { name = "recursive-hello"; })') + + mkdir -p $out/bin + ln -s $hello/bin/hello $out/bin/hello + '' + ``` + + An important restriction on recursive builders is disallowing + arbitrary substitutions. For example, running + + ``` + nix-store -r /nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10 + ``` + + in the above `runCommand` script would be disallowed, as this could + lead to derivations with hidden dependencies or breaking + reproducibility by relying on the current state of the Nix store. An + exception would be if + `/nix/store/kmwd1hq55akdb9sc7l3finr175dajlby-hello-2.10` were + already in the build inputs or built by a previous recursive Nix + call. + )", + }, + { + .tag = Xp::NoUrlLiterals, + .name = "no-url-literals", + .description = R"( + Disallow unquoted URLs as part of the Nix language syntax. The Nix + language allows for URL literals, like so: + + ``` + $ nix repl + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + "http://foo" + ``` + + But enabling this experimental feature will cause the Nix parser to + throw an error when encountering a URL literal: + + ``` + $ nix repl --extra-experimental-features 'no-url-literals' + Welcome to Nix 2.15.0. Type :? for help. + + nix-repl> http://foo + error: URL literals are disabled + + at «string»:1:1: + + 1| http://foo + | ^ + + ``` + + While this is currently an experimental feature, unquoted URLs are + being deprecated and their usage is discouraged. + + The reason is that, as opposed to path literals, URLs have no + special properties that distinguish them from regular strings, URLs + containing parameters have to be quoted anyway, and unquoted URLs + may confuse external tooling. + )", + }, + { + .tag = Xp::FetchClosure, + .name = "fetch-closure", + .description = R"( + Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. + )", + }, + { + .tag = Xp::ReplFlake, + .name = "repl-flake", + .description = R"( + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. + )", + }, + { + .tag = Xp::AutoAllocateUids, + .name = "auto-allocate-uids", + .description = R"( + Allows Nix to automatically pick UIDs for builds, rather than creating + `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + )", + }, + { + .tag = Xp::Cgroups, + .name = "cgroups", + .description = R"( + Allows Nix to execute builds inside cgroups. See + the [`use-cgroups`](#conf-use-cgroups) setting for details. + )", + }, + { + .tag = Xp::DiscardReferences, + .name = "discard-references", + .description = R"( + Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations + that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for + runtime dependencies. + )", + }, +}}; + +static_assert( + []() constexpr { + for (auto [index, feature] : enumerate(xpFeatureDetails)) + if (index != (size_t)feature.tag) + return false; + return true; + }(), + "array order does not match enum tag order"); + const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>; - static auto reverseXpMap = []() - { + static std::unique_ptr<ReverseXpMap> reverseXpMap = []() { auto reverseXpMap = std::make_unique<ReverseXpMap>(); - for (auto & [feature, name] : stringifiedXpFeatures) - (*reverseXpMap)[name] = feature; + for (auto & xpFeature : xpFeatureDetails) + (*reverseXpMap)[xpFeature.name] = xpFeature.tag; return reverseXpMap; }(); @@ -37,20 +217,27 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str return std::nullopt; } -std::string_view showExperimentalFeature(const ExperimentalFeature feature) +std::string_view showExperimentalFeature(const ExperimentalFeature tag) +{ + assert((size_t)tag < xpFeatureDetails.size()); + return xpFeatureDetails[(size_t)tag].name; +} + +nlohmann::json documentExperimentalFeatures() { - const auto ret = get(stringifiedXpFeatures, feature); - assert(ret); - return *ret; + StringMap res; + for (auto & xpFeature : xpFeatureDetails) + res[std::string { xpFeature.name }] = + trim(stripIndentation(xpFeature.description)); + return (nlohmann::json) res; } std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures) { std::set<ExperimentalFeature> res; - for (auto & rawFeature : rawFeatures) { + for (auto & rawFeature : rawFeatures) if (auto feature = parseExperimentalFeature(rawFeature)) res.insert(*feature); - } return res; } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 5948ad7ad..8ef66263a 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -11,8 +11,9 @@ namespace nix { /** * The list of available experimental features. * - * If you update this, don’t forget to also change the map defining their - * string representation in the corresponding `.cc` file. + * If you update this, don’t forget to also change the map defining + * their string representation and documentation in the corresponding + * `.cc` file as well. */ enum struct ExperimentalFeature { @@ -34,26 +35,52 @@ enum struct ExperimentalFeature */ using Xp = ExperimentalFeature; +/** + * Parse an experimental feature (enum value) from its name. Experimental + * feature flag names are hyphenated and do not contain spaces. + */ const std::optional<ExperimentalFeature> parseExperimentalFeature( const std::string_view & name); + +/** + * Show the name of an experimental feature. This is the opposite of + * parseExperimentalFeature(). + */ std::string_view showExperimentalFeature(const ExperimentalFeature); +/** + * Compute the documentation of all experimental features. + * + * See `doc/manual` for how this information is used. + */ +nlohmann::json documentExperimentalFeatures(); + +/** + * Shorthand for `str << showExperimentalFeature(feature)`. + */ std::ostream & operator<<( std::ostream & str, const ExperimentalFeature & feature); /** - * Parse a set of strings to the corresponding set of experimental features, - * ignoring (but warning for) any unkwown feature. + * Parse a set of strings to the corresponding set of experimental + * features, ignoring (but warning for) any unknown feature. */ std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &); +/** + * An experimental feature was required for some (experimental) + * operation, but was not enabled. + */ class MissingExperimentalFeature : public Error { public: + /** + * The experimental feature that was required but not enabled. + */ ExperimentalFeature missingFeature; - MissingExperimentalFeature(ExperimentalFeature); + MissingExperimentalFeature(ExperimentalFeature missingFeature); }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 5735e4715..2c36d9d94 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -1,6 +1,7 @@ #include <iostream> #include <cstring> +#include <openssl/crypto.h> #include <openssl/md5.h> #include <openssl/sha.h> @@ -16,7 +17,6 @@ namespace nix { - static size_t regularHashSize(HashType type) { switch (type) { case htMD5: return md5HashSize; @@ -343,7 +343,7 @@ HashSink::~HashSink() delete ctx; } -void HashSink::write(std::string_view data) +void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); update(ht, *ctx, data); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index be1fdba2a..ae3ee40f4 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -197,7 +197,7 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 576068c22..5aa6bee95 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -183,12 +183,17 @@ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted); -extern Verbosity verbosity; /* suppress msgs > this */ +/** + * suppress msgs > this + */ +extern Verbosity verbosity; -/* Print a message with the standard ErrorInfo format. - In general, use these 'log' macros for reporting problems that may require user - intervention or that need more explanation. Use the 'print' macros for more - lightweight status messages. */ +/** + * Print a message with the standard ErrorInfo format. + * In general, use these 'log' macros for reporting problems that may require user + * intervention or that need more explanation. Use the 'print' macros for more + * lightweight status messages. + */ #define logErrorInfo(level, errorInfo...) \ do { \ if ((level) <= nix::verbosity) { \ @@ -199,9 +204,11 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) #define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) -/* Print a string message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure that the - arguments are evaluated lazily. */ +/** + * Print a string message if the current log level is at least the specified + * level. Note that this has to be implemented as a macro to ensure that the + * arguments are evaluated lazily. + */ #define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7476e6f6c..3d5121a19 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -20,7 +20,7 @@ void BufferedSink::operator () (std::string_view data) buffer size. */ if (bufPos + data.size() >= bufSize) { flush(); - write(data); + writeUnbuffered(data); break; } /* Otherwise, copy the bytes to the buffer. Flush the buffer @@ -38,7 +38,7 @@ void BufferedSink::flush() if (bufPos == 0) return; size_t n = bufPos; bufPos = 0; // don't trigger the assert() in ~BufferedSink() - write({buffer.get(), n}); + writeUnbuffered({buffer.get(), n}); } @@ -48,7 +48,7 @@ FdSink::~FdSink() } -void FdSink::write(std::string_view data) +void FdSink::writeUnbuffered(std::string_view data) { written += data.size(); try { @@ -186,6 +186,22 @@ static DefaultStackAllocator defaultAllocatorSingleton; StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; +std::shared_ptr<void> (*create_coro_gc_hook)() = []() -> std::shared_ptr<void> { + return {}; +}; + +/* This class is used for entry and exit hooks on coroutines */ +class CoroutineContext { + /* Disable GC when entering the coroutine without the boehm patch, + * since it doesn't find the main thread stack in this case. + * std::shared_ptr<void> performs type-erasure, so it will call the right + * deleter. */ + const std::shared_ptr<void> coro_gc_hook = create_coro_gc_hook(); +public: + CoroutineContext() {}; + ~CoroutineContext() {}; +}; + std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun) { struct SourceToSink : FinishSink @@ -206,7 +222,8 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun) if (in.empty()) return; cur = in; - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { LambdaSource source([&](char *out, size_t out_len) { if (cur.empty()) { @@ -223,17 +240,24 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun) }); fun(source); }); + } if (!*coro) { abort(); } - if (!cur.empty()) (*coro)(false); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(false); + } } void finish() override { if (!coro) return; if (!*coro) abort(); - (*coro)(true); + { + CoroutineContext ctx; + (*coro)(true); + } if (*coro) abort(); } }; @@ -264,18 +288,23 @@ std::unique_ptr<Source> sinkToSource( size_t read(char * data, size_t len) override { - if (!coro) + if (!coro) { + CoroutineContext ctx; coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); fun(sink); }); + } if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) (*coro)(); + if (!cur.empty()) { + CoroutineContext ctx; + (*coro)(); + } cur = coro->get(); pos = 0; } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 2cf527023..333c254ea 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -53,7 +53,9 @@ struct BufferedSink : virtual Sink void flush(); - virtual void write(std::string_view data) = 0; +protected: + + virtual void writeUnbuffered(std::string_view data) = 0; }; @@ -133,7 +135,7 @@ struct FdSink : BufferedSink ~FdSink(); - void write(std::string_view data) override; + void writeUnbuffered(std::string_view data) override; bool good() override; @@ -520,7 +522,7 @@ struct FramedSink : nix::BufferedSink } } - void write(std::string_view data) override + void writeUnbuffered(std::string_view data) override { /* Don't send more data if the remote has encountered an error. */ @@ -551,4 +553,10 @@ struct StackAllocator { static StackAllocator *defaultAllocator; }; +/* Disabling GC when entering a coroutine (without the boehm patch). + mutable to avoid boehm gc dependency in libutil. + */ +extern std::shared_ptr<void> (*create_coro_gc_hook)(); + + } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 8be6730dd..f250e934e 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -156,12 +156,54 @@ namespace nix { } TEST(Config, toJSONOnNonEmptyConfig) { + using nlohmann::literals::operator "" _json; Config config; - std::map<std::string, Config::SettingInfo> settings; - Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + Setting<std::string> setting{ + &config, + "", + "name-of-the-setting", + "description", + }; + setting.assign("value"); + + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": null + } + })#"_json); + } + + TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) { + using nlohmann::literals::operator "" _json; + Config config; + Setting<std::string> setting{ + &config, + "", + "name-of-the-setting", + "description", + {}, + true, + Xp::Flakes, + }; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); + ASSERT_EQ(config.toJSON(), + R"#({ + "name-of-the-setting": { + "aliases": [], + "defaultValue": "", + "description": "description\n", + "documentDefault": true, + "value": "value", + "experimentalFeature": "flakes" + } + })#"_json); } TEST(Config, setSettingAlias) { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 14b32279c..0e09fae97 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -40,15 +40,15 @@ public: /** * Execute work items until the queue is empty. - * - * \note Note that work items are allowed to add new items to the - * queue; this is handled correctly. * - * Queue processing stops prematurely if any work item throws an - * exception. This exception is propagated to the calling thread. If - * multiple work items throw an exception concurrently, only one - * item is propagated; the others are printed on stderr and - * otherwise ignored. + * \note Note that work items are allowed to add new items to the + * queue; this is handled correctly. + * + * Queue processing stops prematurely if any work item throws an + * exception. This exception is propagated to the calling thread. If + * multiple work items throw an exception concurrently, only one + * item is propagated; the others are printed on stderr and + * otherwise ignored. */ void process(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 843a10eab..21d1c8dcd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -47,6 +47,9 @@ extern char * * environ __attribute__((weak)); namespace nix { +void initLibUtil() { +} + std::optional<std::string> getEnv(const std::string & key) { char * value = getenv(key.c_str()); @@ -1744,14 +1747,40 @@ void triggerInterrupt() } static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; -void startSignalHandlerThread() +void setChildSignalMask(sigset_t * sigs) { - updateWindowSize(); + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} +void saveSignalMask() { if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) throw SysError("querying signal mask"); + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + sigset_t set; sigemptyset(&set); sigaddset(&set, SIGINT); @@ -1767,6 +1796,20 @@ void startSignalHandlerThread() static void restoreSignals() { + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) throw SysError("restoring signals"); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6c2706cc1..040fed68f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -32,78 +32,114 @@ namespace nix { struct Sink; struct Source; +void initLibUtil(); -/* The system for which Nix is compiled. */ +/** + * The system for which Nix is compiled. + */ extern const std::string nativeSystem; -/* Return an environment variable. */ +/** + * @return an environment variable. + */ std::optional<std::string> getEnv(const std::string & key); -/* Return a non empty environment variable. Returns nullopt if the env -variable is set to "" */ +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ std::optional<std::string> getEnvNonEmpty(const std::string & key); -/* Get the entire environment. */ +/** + * Get the entire environment. + */ std::map<std::string, std::string> getEnv(); -/* Clear the environment. */ +/** + * Clear the environment. + */ void clearEnv(); -/* Return an absolutized path, resolving paths relative to the - specified directory, or the current directory otherwise. The path - is also canonicalised. */ +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ Path absPath(Path path, std::optional<PathView> dir = {}, bool resolveSymlinks = false); -/* Canonicalise a path by removing all `.' or `..' components and - double or trailing slashes. Optionally resolves all symlink - components such that each component of the resulting path is *not* - a symbolic link. */ +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ Path canonPath(PathView path, bool resolveSymlinks = false); -/* Return the directory part of the given canonical path, i.e., - everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means `/' - is returned.*/ +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ Path dirOf(const PathView path); -/* Return the base name of the given canonical path, i.e., everything - following the final `/' (trailing slashes are removed). */ +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ std::string_view baseNameOf(std::string_view path); -/* Perform tilde expansion on a path. */ +/** + * Perform tilde expansion on a path. + */ std::string expandTilde(std::string_view path); -/* Check whether 'path' is a descendant of 'dir'. Both paths must be - canonicalized. */ +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ bool isInDir(std::string_view path, std::string_view dir); -/* Check whether 'path' is equal to 'dir' or a descendant of - 'dir'. Both paths must be canonicalized. */ +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ bool isDirOrInDir(std::string_view path, std::string_view dir); -/* Get status of `path'. */ +/** + * Get status of `path`. + */ struct stat stat(const Path & path); struct stat lstat(const Path & path); -/* Return true iff the given path exists. */ +/** + * @return true iff the given path exists. + */ bool pathExists(const Path & path); -/* Read the contents (target) of a symbolic link. The result is not - in any way canonicalised. */ +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ Path readLink(const Path & path); bool isLink(const Path & path); -/* Read the contents of a directory. The entries `.' and `..' are - removed. */ +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ struct DirEntry { std::string name; ino_t ino; - unsigned char type; // one of DT_* + /** + * one of DT_* + */ + unsigned char type; DirEntry(std::string name, ino_t ino, unsigned char type) : name(std::move(name)), ino(ino), type(type) { } }; @@ -114,74 +150,110 @@ DirEntries readDirectory(const Path & path); unsigned char getFileType(const Path & path); -/* Read the contents of a file into a string. */ +/** + * Read the contents of a file into a string. + */ std::string readFile(int fd); std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); -/* Write a string to a file. */ +/** + * Write a string to a file. + */ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); -/* Flush a file's parent directory to disk */ +/** + * Flush a file's parent directory to disk + */ void syncParent(const Path & path); -/* Read a line from a file descriptor. */ +/** + * Read a line from a file descriptor. + */ std::string readLine(int fd); -/* Write a line to a file descriptor. */ +/** + * Write a line to a file descriptor. + */ void writeLine(int fd, std::string s); -/* Delete a path; i.e., in the case of a directory, it is deleted - recursively. It's not an error if the path does not exist. The - second variant returns the number of bytes and blocks freed. */ +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ void deletePath(const Path & path); void deletePath(const Path & path, uint64_t & bytesFreed); std::string getUserName(); -/* Return the given user's home directory from /etc/passwd. */ +/** + * @return the given user's home directory from /etc/passwd. + */ Path getHomeOf(uid_t userId); -/* Return $HOME or the user's home directory from /etc/passwd. */ +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ Path getHome(); -/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ Path getCacheDir(); -/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ Path getConfigDir(); -/* Return the directories to search for user configuration files */ +/** + * @return the directories to search for user configuration files + */ std::vector<Path> getConfigDirs(); -/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ Path getDataDir(); -/* Return the path of the current executable. */ +/** + * @return the path of the current executable. + */ std::optional<Path> getSelfExe(); -/* Return $XDG_STATE_HOME or $HOME/.local/state. */ +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ Path getStateDir(); -/* Create the Nix state directory and return the path to it. */ +/** + * Create the Nix state directory and return the path to it. + */ Path createNixStateDir(); -/* Create a directory and all its parents, if necessary. Returns the - list of created directories, in order of creation. */ +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ Paths createDirs(const Path & path); inline Paths createDirs(PathView path) { return createDirs(Path(path)); } -/* Create a symlink. */ +/** + * Create a symlink. + */ void createSymlink(const Path & target, const Path & link, std::optional<time_t> mtime = {}); -/* Atomically create or replace a symlink. */ +/** + * Atomically create or replace a symlink. + */ void replaceSymlink(const Path & target, const Path & link, std::optional<time_t> mtime = {}); @@ -197,24 +269,32 @@ void renameFile(const Path & src, const Path & dst); void moveFile(const Path & src, const Path & dst); -/* Wrappers arount read()/write() that read/write exactly the - requested number of bytes. */ +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ void readFull(int fd, char * buf, size_t count); void writeFull(int fd, std::string_view s, bool allowInterrupts = true); MakeError(EndOfFile, Error); -/* Read a file descriptor until EOF occurs. */ +/** + * Read a file descriptor until EOF occurs. + */ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); -/* If cgroups are active, attempt to calculate the number of CPUs available. - If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ unsigned int getMaxCPU(); -/* Automatic cleanup of resources. */ +/** + * Automatic cleanup of resources. + */ class AutoDelete @@ -252,11 +332,15 @@ public: }; -/* Create a temporary directory. */ +/** + * Create a temporary directory. + */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); -/* Create a temporary file, returning a file handle and its path. */ +/** + * Create a temporary file, returning a file handle and its path. + */ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix"); @@ -299,27 +383,36 @@ public: }; -/* Kill all processes running under the specified uid by sending them - a SIGKILL. */ +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ void killUser(uid_t uid); -/* Fork a process that runs the given function, and return the child - pid to the caller. */ +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ struct ProcessOptions { std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; bool allowVfork = false; - int cloneFlags = 0; // use clone() with the specified flags (Linux only) + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; }; pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); -/* Run a program and return its stdout in a string (i.e., like the - shell backtick operator). */ +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional<std::string> & input = {}); @@ -344,25 +437,37 @@ std::pair<int, std::string> runProgram(RunOptions && options); void runProgram2(const RunOptions & options); -/* Change the stack size. */ +/** + * Change the stack size. + */ void setStackSize(size_t stackSize); -/* Restore the original inherited Unix process context (such as signal - masks, stack size). */ +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ void restoreProcessContext(bool restoreMounts = true); -/* Save the current mount namespace. Ignored if called more than - once. */ +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ void saveMountNamespace(); -/* Restore the mount namespace saved by saveMountNamespace(). Ignored - if saveMountNamespace() was never called. */ +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ void restoreMountNamespace(); -/* Cause this thread to not share any FS attributes with the main - thread, because this causes setns() in restoreMountNamespace() to - fail. */ +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ void unshareFilesystem(); @@ -377,16 +482,22 @@ public: { } }; -/* Convert a list of strings to a null-terminated vector of char - *'s. The result must not be accessed beyond the lifetime of the - list of strings. */ +/** + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. + */ std::vector<char *> stringsToCharPtrs(const Strings & ss); -/* Close all file descriptors except those listed in the given set. - Good practice in child processes. */ +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ void closeMostFDs(const std::set<int> & exceptions); -/* Set the close-on-exec flag for the given file descriptor. */ +/** + * Set the close-on-exec flag for the given file descriptor. + */ void closeOnExec(int fd); @@ -412,12 +523,16 @@ MakeError(Interrupted, BaseError); MakeError(FormatError, Error); -/* String tokenizer. */ +/** + * String tokenizer. + */ template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); -/* Concatenate the given strings with a separator between the - elements. */ +/** + * Concatenate the given strings with a separator between the + * elements. + */ template<class C> std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -442,7 +557,9 @@ auto concatStrings(Parts && ... parts) } -/* Add quotes around a collection of strings. */ +/** + * Add quotes around a collection of strings. + */ template<class C> Strings quoteStrings(const C & c) { Strings res; @@ -451,16 +568,23 @@ template<class C> Strings quoteStrings(const C & c) return res; } -/* Remove trailing whitespace from a string. FIXME: return - std::string_view. */ +/** + * Remove trailing whitespace from a string. + * + * \todo return std::string_view. + */ std::string chomp(std::string_view s); -/* Remove whitespace from the start and end of a string. */ +/** + * Remove whitespace from the start and end of a string. + */ std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); -/* Replace all occurrences of a string inside another string. */ +/** + * Replace all occurrences of a string inside another string. + */ std::string replaceStrings( std::string s, std::string_view from, @@ -470,14 +594,18 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/* Convert the exit status of a child as returned by wait() into an - error string. */ +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ std::string statusToString(int status); bool statusOk(int status); -/* Parse a string into an integer. */ +/** + * Parse a string into an integer. + */ template<class N> std::optional<N> string2Int(const std::string_view s) { @@ -490,8 +618,10 @@ std::optional<N> string2Int(const std::string_view s) } } -/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or - 'T' denoting a binary unit prefix. */ +/** + * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + * 'T' denoting a binary unit prefix. + */ template<class N> N string2IntWithUnitPrefix(std::string_view s) { @@ -512,7 +642,9 @@ N string2IntWithUnitPrefix(std::string_view s) throw UsageError("'%s' is not an integer", s); } -/* Parse a string into a float. */ +/** + * Parse a string into a float. + */ template<class N> std::optional<N> string2Float(const std::string_view s) { @@ -524,7 +656,9 @@ std::optional<N> string2Float(const std::string_view s) } -/* Convert a little-endian integer to host order. */ +/** + * Convert a little-endian integer to host order. + */ template<typename T> T readLittleEndian(unsigned char * p) { @@ -536,66 +670,90 @@ T readLittleEndian(unsigned char * p) } -/* Return true iff `s' starts with `prefix'. */ +/** + * @return true iff `s` starts with `prefix`. + */ bool hasPrefix(std::string_view s, std::string_view prefix); -/* Return true iff `s' ends in `suffix'. */ +/** + * @return true iff `s` ends in `suffix`. + */ bool hasSuffix(std::string_view s, std::string_view suffix); -/* Convert a string to lower case. */ +/** + * Convert a string to lower case. + */ std::string toLower(const std::string & s); -/* Escape a string as a shell word. */ +/** + * Escape a string as a shell word. + */ std::string shellEscape(const std::string_view s); -/* Exception handling in destructors: print an error message, then - ignore the exception. */ +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + */ void ignoreException(Verbosity lvl = lvlError); -/* Tree formatting. */ +/** + * Tree formatting. + */ constexpr char treeConn[] = "├───"; constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/* Determine whether ANSI escape sequences are appropriate for the - present output. */ +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ bool shouldANSI(); -/* Truncate a string to 'width' printable characters. If 'filterAll' - is true, all ANSI escape sequences are filtered out. Otherwise, - some escape sequences (such as colour setting) are copied but not - included in the character count. Also, tabs are expanded to - spaces. */ +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits<unsigned int>::max()); -/* Base64 encoding/decoding. */ +/** + * Base64 encoding/decoding. + */ std::string base64Encode(std::string_view s); std::string base64Decode(std::string_view s); -/* Remove common leading whitespace from the lines in the string - 's'. For example, if every line is indented by at least 3 spaces, - then we remove 3 spaces from the start of every line. */ +/** + * Remove common leading whitespace from the lines in the string + * 's'. For example, if every line is indented by at least 3 spaces, + * then we remove 3 spaces from the start of every line. + */ std::string stripIndentation(std::string_view s); -/* Get the prefix of 's' up to and excluding the next line break (LF - optionally preceded by CR), and the remainder following the line - break. */ +/** + * Get the prefix of 's' up to and excluding the next line break (LF + * optionally preceded by CR), and the remainder following the line + * break. + */ std::pair<std::string_view, std::string_view> getLine(std::string_view s); -/* Get a value for the specified key from an associate container. */ +/** + * Get a value for the specified key from an associate container. + */ template <class T> const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { @@ -612,7 +770,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) return &i->second; } -/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ +/** + * Get a value for the specified key from an associate container, or a default value if the key isn't present. + */ template <class T> const typename T::mapped_type & getOr(T & map, const typename T::key_type & key, @@ -623,7 +783,9 @@ const typename T::mapped_type & getOr(T & map, return i->second; } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template <class T> std::optional<typename T::value_type> remove_begin(T & c) { @@ -635,7 +797,9 @@ std::optional<typename T::value_type> remove_begin(T & c) } -/* Remove and return the first item from a container. */ +/** + * Remove and return the first item from a container. + */ template <class T> std::optional<typename T::value_type> pop(T & c) { @@ -650,25 +814,48 @@ template<typename T> class Callback; -/* Start a thread that handles various signals. Also block those signals - on the current thread (and thus any threads created by it). */ +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ void startSignalHandlerThread(); +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + struct InterruptCallback { virtual ~InterruptCallback() { }; }; -/* Register a function that gets called on SIGINT (in a non-signal - context). */ +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ std::unique_ptr<InterruptCallback> createInterruptCallback( std::function<void()> callback); void triggerInterrupt(); -/* A RAII class that causes the current thread to receive SIGUSR1 when - the signal handler thread receives SIGINT. That is, this allows - SIGINT to be multiplexed to multiple threads. */ +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ struct ReceiveInterrupts { pthread_t target; @@ -682,8 +869,10 @@ struct ReceiveInterrupts -/* A RAII helper that increments a counter on construction and - decrements it on destruction. */ +/** + * A RAII helper that increments a counter on construction and + * decrements it on destruction. + */ template<typename T> struct MaintainCount { @@ -694,33 +883,50 @@ struct MaintainCount }; -/* Return the number of rows and columns of the terminal. */ +/** + * @return the number of rows and columns of the terminal. + */ std::pair<unsigned short, unsigned short> getWindowSize(); -/* Used in various places. */ +/** + * Used in various places. + */ typedef std::function<bool(const Path & path)> PathFilter; extern PathFilter defaultPathFilter; -/* Common initialisation performed in child processes. */ +/** + * Common initialisation performed in child processes. + */ void commonChildInit(); -/* Create a Unix domain socket. */ +/** + * Create a Unix domain socket. + */ AutoCloseFD createUnixDomainSocket(); -/* Create a Unix domain socket in listen mode. */ +/** + * Create a Unix domain socket in listen mode. + */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); -/* Bind a Unix domain socket to a path. */ +/** + * Bind a Unix domain socket to a path. + */ void bind(int fd, const std::string & path); -/* Connect to a Unix domain socket. */ +/** + * Connect to a Unix domain socket. + */ void connect(int fd, const std::string & path); -// A Rust/Python-like enumerate() iterator adapter. -// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +/** + * A Rust/Python-like enumerate() iterator adapter. + * + * Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. + */ template <typename T, typename TIter = decltype(std::begin(std::declval<T>())), typename = decltype(std::end(std::declval<T>()))> @@ -730,23 +936,25 @@ constexpr auto enumerate(T && iterable) { size_t i; TIter iter; - bool operator != (const iterator & other) const { return iter != other.iter; } - void operator ++ () { ++i; ++iter; } - auto operator * () const { return std::tie(i, *iter); } + constexpr bool operator != (const iterator & other) const { return iter != other.iter; } + constexpr void operator ++ () { ++i; ++iter; } + constexpr auto operator * () const { return std::tie(i, *iter); } }; struct iterable_wrapper { T iterable; - auto begin() { return iterator{ 0, std::begin(iterable) }; } - auto end() { return iterator{ 0, std::end(iterable) }; } + constexpr auto begin() { return iterator{ 0, std::begin(iterable) }; } + constexpr auto end() { return iterator{ 0, std::end(iterable) }; } }; return iterable_wrapper{ std::forward<T>(iterable) }; } -// C++17 std::visit boilerplate +/** + * C++17 std::visit boilerplate + */ template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; @@ -754,8 +962,10 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; std::string showBytes(uint64_t bytes); -/* Provide an addition operator between strings and string_views - inexplicably omitted from the standard library. */ +/** + * Provide an addition operator between strings and string_views + * inexplicably omitted from the standard library. + */ inline std::string operator + (const std::string & s1, std::string_view s2) { auto s = s1; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 7035e6a7b..03fc93962 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -204,10 +204,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -218,7 +218,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string hash = *i++; std::string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + .hash = { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + .references = {}, + }))); } @@ -964,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); - info.ca = parseContentAddressOpt(readString(in)); + info.ca = ContentAddress::parseOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5168413d2..16e48a39b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -42,14 +42,18 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + std::move(*namePart), + FixedOutputInfo { + .hash = { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + .references = {}, + }, narHash, }; info.narSize = sink.s.size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource(sink.s); diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 7e4a7ba86..7ae7b4ea6 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,5 @@ +///@file + #include "command.hh" #include "shared.hh" #include "local-store.hh" @@ -34,6 +36,19 @@ using namespace nix; using namespace nix::daemon; +/** + * Settings related to authenticating clients for the Nix daemon. + * + * For pipes we have little good information about the client side, but + * for Unix domain sockets we do. So currently these options implemented + * mandatory access control based on user names and group names (looked + * up and translated to UID/GIDs in the CLI process that runs the code + * in this file). + * + * No code outside of this file knows about these settings (this is not + * exposed in a header); all authentication and authorization happens in + * `daemon.cc`. + */ struct AuthorizationSettings : Config { Setting<Strings> trustedUsers{ @@ -54,7 +69,9 @@ struct AuthorizationSettings : Config { > directories that are otherwise inacessible to them. )"}; - /* ?Who we trust to use the daemon in safe ways */ + /** + * Who we trust to use the daemon in safe ways + */ Setting<Strings> allowedUsers{ this, {"*"}, "allowed-users", R"( @@ -112,8 +129,36 @@ static void setSigChldAction(bool autoReap) throw SysError("setting SIGCHLD handler"); } +/** + * @return Is the given user a member of this group? + * + * @param user User specified by username. + * + * @param group Group the user might be a member of. + */ +static bool matchUser(std::string_view user, const struct group & gr) +{ + for (char * * mem = gr.gr_mem; *mem; mem++) + if (user == std::string_view(*mem)) return true; + return false; +} + -bool matchUser(const std::string & user, const std::string & group, const Strings & users) +/** + * Does the given user (specified by user name and primary group name) + * match the given user/group whitelist? + * + * If the list allows all users: Yes. + * + * If the username is in the set: Yes. + * + * If the groupname is in the set: Yes. + * + * If the user is in another group which is in the set: yes. + * + * Otherwise: No. + */ +static bool matchUser(const std::string & user, const std::string & group, const Strings & users) { if (find(users.begin(), users.end(), "*") != users.end()) return true; @@ -126,8 +171,7 @@ bool matchUser(const std::string & user, const std::string & group, const String if (group == i.substr(1)) return true; struct group * gr = getgrnam(i.c_str() + 1); if (!gr) continue; - for (char * * mem = gr->gr_mem; *mem; mem++) - if (user == std::string(*mem)) return true; + if (matchUser(user, *gr)) return true; } return false; @@ -145,7 +189,9 @@ struct PeerInfo }; -// Get the identity of the caller, if possible. +/** + * Get the identity of the caller, if possible. + */ static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -179,6 +225,9 @@ static PeerInfo getPeerInfo(int remote) #define SD_LISTEN_FDS_START 3 +/** + * Open a store without a path info cache. + */ static ref<Store> openUncachedStore() { Store::Params params; // FIXME: get params from somewhere @@ -187,7 +236,44 @@ static ref<Store> openUncachedStore() return openStore(settings.storeUri, params); } +/** + * Authenticate a potential client + * + * @param peer Information about other end of the connection, the client which + * wants to communicate with us. + * + * @return A pair of a `TrustedFlag`, whether the potential client is trusted, + * and the name of the user (useful for printing messages). + * + * If the potential client is not allowed to talk to us, we throw an `Error`. + */ +static std::pair<TrustedFlag, std::string> authPeer(const PeerInfo & peer) +{ + TrustedFlag trusted = NotTrusted; + + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + std::string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + std::string group = gr ? gr->gr_name : std::to_string(peer.gid); + + const Strings & trustedUsers = authorizationSettings.trustedUsers; + const Strings & allowedUsers = authorizationSettings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + return { trusted, std::move(user) }; +} + +/** + * Run a server. The loop opens a socket and accepts new connections from that + * socket. + */ static void daemonLoop() { if (chdir("/") == -1) @@ -231,23 +317,9 @@ static void daemonLoop() closeOnExec(remote.get()); - TrustedFlag trusted = NotTrusted; PeerInfo peer = getPeerInfo(remote.get()); - - struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; - std::string user = pw ? pw->pw_name : std::to_string(peer.uid); - - struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; - std::string group = gr ? gr->gr_name : std::to_string(peer.gid); - - Strings trustedUsers = authorizationSettings.trustedUsers; - Strings allowedUsers = authorizationSettings.allowedUsers; - - if (matchUser(user, group, trustedUsers)) - trusted = Trusted; - - if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + auto [_trusted, user] = authPeer(peer); + auto trusted = _trusted; printInfo((std::string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : ""), peer.pidKnown ? std::to_string(peer.pid) : "<unknown>", @@ -294,45 +366,74 @@ static void daemonLoop() } } +/** + * Forward a standard IO connection to the given remote store. + * + * We just act as a middleman blindly ferry output between the standard + * input/output and the remote store connection, not processing anything. + * + * Loops until standard input disconnects, or an error is encountered. + */ +static void forwardStdioConnection(RemoteStore & store) { + auto conn = store.openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } +} + +/** + * Process a client connecting to us via standard input/output + * + * Unlike `forwardStdioConnection()` we do process commands ourselves in + * this case, not delegating to another daemon. + * + * @note `Trusted` is unconditionally passed because in this mode we + * blindly trust the standard streams. Limiting access to those is + * explicitly not `nix-daemon`'s responsibility. + */ +static void processStdioConnection(ref<Store> store) +{ + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + processConnection(store, from, to, Trusted, NotRecursive); +} + +/** + * Entry point shared between the new CLI `nix daemon` and old CLI + * `nix-daemon`. + */ static void runDaemon(bool stdio) { if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) { - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limiting access to those is explicitly - not `nix-daemon`'s responsibility. */ - processConnection(openUncachedStore(), from, to, Trusted, NotRecursive); - } + auto store = openUncachedStore(); + + if (auto remoteStore = store.dynamic_pointer_cast<RemoteStore>()) + forwardStdioConnection(*remoteStore); + else + processStdioConnection(store); } else daemonLoop(); } diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc new file mode 100644 index 000000000..4d91d4538 --- /dev/null +++ b/src/nix/derivation-add.cc @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include <nlohmann/json.hpp> + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref<Store> store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2<CmdAddDerivation>({"derivation", "add"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md new file mode 100644 index 000000000..f116681ab --- /dev/null +++ b/src/nix/derivation-add.md @@ -0,0 +1,18 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +[store derivation]: ../../glossary.md#gloss-store-derivation + +The JSON format is documented under the [`derivation show`] command. + +[`derivation show`]: ./nix3-derivation-show.md + +)"" diff --git a/src/nix/show-derivation.cc b/src/nix/derivation-show.cc index 4a406ae08..bf637246d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/derivation-show.cc @@ -1,5 +1,5 @@ // FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? #include "command.hh" #include "common-args.hh" @@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand std::string doc() override { return - #include "show-derivation.md" + #include "derivation-show.md" ; } @@ -61,4 +61,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto rCmdShowDerivation = registerCommand<CmdShowDerivation>("show-derivation"); +static auto rCmdShowDerivation = registerCommand2<CmdShowDerivation>({"derivation", "show"}); diff --git a/src/nix/show-derivation.md b/src/nix/derivation-show.md index 1d37c6f5a..1296e2885 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/derivation-show.md @@ -8,7 +8,7 @@ R""( [store derivation]: ../../glossary.md#gloss-store-derivation ```console - # nix show-derivation nixpkgs#hello + # nix derivation show nixpkgs#hello { "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { … @@ -20,14 +20,14 @@ R""( NixOS system: ```console - # nix show-derivation -r /run/current-system + # nix derivation show -r /run/current-system ``` * Print all files fetched using `fetchurl` by Firefox's dependency graph: ```console - # nix show-derivation -r nixpkgs#firefox \ + # nix derivation show -r nixpkgs#firefox \ | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ | uniq | sort ``` @@ -39,10 +39,11 @@ R""( # Description This command prints on standard output a JSON representation of the -[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. By default, this command only shows top-level derivations, but with `--recursive`, it also shows their dependencies. @@ -51,6 +52,9 @@ The JSON output is a JSON object whose keys are the store paths of the derivations, and whose values are a JSON object with the following fields: +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc new file mode 100644 index 000000000..cd3975a4f --- /dev/null +++ b/src/nix/derivation.cc @@ -0,0 +1,25 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : virtual NixMultiCommand +{ + CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix derivation' requires a sub-command."); + command->second->run(); + } +}; + +static auto rCmdDerivation = registerCommand<CmdDerivation>("derivation"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 354b03cf6..1aa6831d3 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -33,6 +33,10 @@ bool checkFail(const std::string & msg) { return false; } +void checkInfo(const std::string & msg) { + notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); +} + } struct CmdDoctor : StoreCommand @@ -63,6 +67,7 @@ struct CmdDoctor : StoreCommand success &= checkProfileRoots(store); } success &= checkStoreProtocol(store->getProtocol()); + checkTrustedUser(store); if (!success) throw Exit(2); @@ -138,6 +143,14 @@ struct CmdDoctor : StoreCommand return checkPass("Client protocol matches store protocol."); } + + void checkTrustedUser(ref<Store> store) + { + std::string_view trusted = store->isTrustedClient() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } }; static auto rCmdDoctor = registerCommand<CmdDoctor>("doctor"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 4d4164333..705061d25 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -127,6 +127,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, + {"show-derivation", {"derivation", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, @@ -374,6 +375,11 @@ void mainWrapped(int argc, char * * argv) return; } + if (argc == 2 && std::string(argv[1]) == "__dump-xp-features") { + logger->cout(documentExperimentalFeatures().dump()); + return; + } + Finally printCompletions([&]() { if (completions) { diff --git a/src/nix/nix.md b/src/nix/nix.md index e1865b31c..1ef6c7fcd 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,12 +48,17 @@ manual](https://nixos.org/manual/nix/stable/). # Installables +> **Warning** \ +> Installables are part of the unstable +> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> and subject to change without notice. + Many `nix` subcommands operate on one or more *installables*. These are command line arguments that represent something that can be realised in the Nix store. The following types of installable are supported by most commands: -- [Flake output attribute](#flake-output-attribute) +- [Flake output attribute](#flake-output-attribute) (experimental) - [Store path](#store-path) - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path @@ -63,6 +68,13 @@ That is, Nix will operate on the default flake output attribute of the flake in ### Flake output attribute +> **Warning** \ +> Flake output attribute installables depend on both the +> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> and +> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> experimental features, and subject to change without notice. + Example: `nixpkgs#hello` These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 5c44510ab..ec450e8e0 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -28,15 +28,20 @@ struct CmdPingStore : StoreCommand, MixJSON store->connect(); if (auto version = store->getVersion()) notice("Version: %s", *version); + if (auto trusted = store->isTrustedClient()) + notice("Trusted: %s", *trusted); } else { nlohmann::json res; Finally printRes([&]() { logger->cout("%s", res); }); + res["url"] = store->getUri(); store->connect(); if (auto version = store->getVersion()) res["version"] = *version; + if (auto trusted = store->isTrustedClient()) + res["trusted"] = *trusted; } } }; diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 51c8a3319..56e7bbb6e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -67,7 +67,13 @@ std::tuple<StorePath, Hash> prefetchFile( the store. */ if (expectedHash) { hashType = expectedHash->type; - storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { + .hash = { + .method = ingestionMethod, + .hash = *expectedHash, + }, + .references = {}, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -118,7 +124,7 @@ std::tuple<StorePath, Hash> prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = getContentAddressHash(*info.ca); + hash = info.ca->getHash(); } return {storePath.value(), hash.value()}; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 88d4a3ce5..fd63b3519 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -200,12 +200,22 @@ struct ProfileManifest auto narHash = hashString(htSHA256, sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + "profile", + FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + .references = { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, + }, narHash, }; - info.references = std::move(references); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; StringSource source(sink.s); store->addToStore(info, source); diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 98e1c5125..7754ad276 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) -nix show-derivation "$drv" --arg seed 1 +nix derivation show "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh new file mode 100644 index 000000000..3615177e9 --- /dev/null +++ b/tests/ca/derivation-json.sh @@ -0,0 +1,26 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json +drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json +diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json diff --git a/tests/derivation-json.sh b/tests/derivation-json.sh new file mode 100644 index 000000000..b6be5d977 --- /dev/null +++ b/tests/derivation-json.sh @@ -0,0 +1,12 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/experimental-features.sh b/tests/experimental-features.sh index a4d55f5f4..73554da8c 100644 --- a/tests/experimental-features.sh +++ b/tests/experimental-features.sh @@ -1,25 +1,27 @@ source common.sh -# Without flakes, flake options should not show up -# With flakes, flake options should show up - -function both_ways { - nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake - nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake - - # Also, the order should not matter - nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake - nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake -} - -# Simple case, the configuration effects the running command -both_ways show-config - -# Skipping for now, because we actually *do* want these to show up in -# the manual, just be marked experimental. Will reenable once the manual -# generation takes advantage of the JSON metadata on this. - -# both_ways store gc --help +# Skipping these two for now, because we actually *do* want flags and +# config settings to always show up in the manual, just be marked +# experimental. Will reenable once the manual generation takes advantage +# of the JSON metadata on this. +# +# # Without flakes, flake options should not show up +# # With flakes, flake options should show up +# +# function grep_both_ways { +# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake +# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# +# # Also, the order should not matter +# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake +# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# } +# +# # Simple case, the configuration effects the running command +# grep_both_ways show-config +# +# # Medium case, the configuration effects --help +# grep_both_ways store gc --help expect 1 nix --experimental-features 'nix-command' show-config --flake-registry 'https://no' nix --experimental-features 'nix-command flakes' show-config --flake-registry 'https://no' diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index 7595fdd35..c7dadf397 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -37,8 +37,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) diff --git a/tests/legacy-ssh-store.sh b/tests/legacy-ssh-store.sh new file mode 100644 index 000000000..71b716b84 --- /dev/null +++ b/tests/legacy-ssh-store.sh @@ -0,0 +1,4 @@ +source common.sh + +# Check that store ping trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff --git a/tests/local-store.sh b/tests/local-store.sh index 0247346f1..89502f864 100644 --- a/tests/local-store.sh +++ b/tests/local-store.sh @@ -17,3 +17,6 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] + +# Ensure store ping trusted works with local store +nix --store ./x store ping --json | jq -e '.trusted' diff --git a/tests/local.mk b/tests/local.mk index ccd76eeac..6cb466e8e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -17,6 +17,7 @@ nix_tests = \ ca/gc.sh \ gc.sh \ remote-store.sh \ + legacy-ssh-store.sh \ lang.sh \ experimental-features.sh \ fetchMercurial.sh \ @@ -101,6 +102,8 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ ca/why-depends.sh \ + derivation-json.sh \ + ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ nix_path.sh \ diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 652e8a8f2..4ef5b484a 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -144,6 +144,7 @@ expect 1 nix profile install $flake2Dir diff -u <( nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ | grep -vE "^warning: " \ + | grep -vE "^error \(ignored\): " \ || true ) <(cat << EOF error: An existing package already provides the following file: diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 1ae126794..ea32a20d3 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -5,8 +5,19 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor +# Ensure that store ping trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' + startDaemon +if isDaemonNewer "2.15pre0"; then + # Ensure that ping works trusted with new daemon + nix store ping --json | jq -e '.trusted' +else + # And the the field is absent with the old daemon + nix store ping --json | jq -e 'has("trusted") | not' +fi + # Test import-from-derivation through the daemon. [[ $(nix eval --impure --raw --expr ' with import ./config.nix; diff --git a/tests/repl.sh b/tests/repl.sh index be8adb742..2b3789521 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -79,6 +79,14 @@ testReplResponse ' "result: ${a}" ' "result: 2" +# check dollar escaping https://github.com/NixOS/nix/issues/4909 +# note the escaped \, +# \\ +# because the second argument is a regex +testReplResponse ' +"$" + "{hi}" +' '"\\${hi}"' + testReplResponse ' drvPath ' '".*-simple.drv"' \ |