diff options
251 files changed, 10251 insertions, 3632 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 3372b1f03..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,27 +0,0 @@ -<!-- - -# Filing a Nix issue - -*WAIT* Are you sure you're filing your issue in the right repository? - -We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time. - -This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*. - -If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues. - -Examples of _Nix_ issues: - -- Nix segfaults when I run `nix-build -A blahblah` -- The Nix language needs a new builtin: `builtins.foobar` -- Regression in the behavior of `nix-env` in Nix 2.0 - -Examples of _nixpkgs_ issues: - -- glibc is b0rked on aarch64 -- chromium in NixOS doesn't support U2F but google-chrome does! -- The OpenJDK package on macOS is missing a key component - -Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly. - ---> diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..e6d346bc1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** + +A clear and concise description of what the bug is. + +If you have a problem with a specific package or NixOS, +you probably want to file an issue at https://github.com/NixOS/nixpkgs/issues. + +**Steps To Reproduce** + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** + +A clear and concise description of what you expected to happen. + +**`nix-env --version` output** + +**Additional context** + +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..392ed30c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: improvement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 87997414d..7feefc855 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ jobs: tests: strategy: matrix: - os: [ubuntu-18.04, macos] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index e10c75418..983026570 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,10 @@ perl/Makefile.config /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/*.gen.hh +*.gen.* + +# /src/libutil/ +/src/libutil/tests/libutil-tests /src/nix/nix @@ -75,6 +78,8 @@ perl/Makefile.config /src/nix-copy-closure/nix-copy-closure +/src/error-demo/error-demo + /src/build-remote/build-remote # /tests/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ee4ea1ac6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -matrix: - include: - - language: osx - script: ./tests/install-darwin.sh - - language: nix - script: nix-build release.nix -A build.x86_64-linux -notifications: - email: false @@ -1,9 +1,10 @@ makefiles = \ mk/precompiled-headers.mk \ local.mk \ - nix-rust/local.mk \ src/libutil/local.mk \ + src/libutil/tests/local.mk \ src/libstore/local.mk \ + src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ src/nix/local.mk \ diff --git a/Makefile.config.in b/Makefile.config.in index fe609ce06..b632444e8 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -1,42 +1,44 @@ AR = @AR@ BDW_GC_LIBS = @BDW_GC_LIBS@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ -LDFLAGS = @LDFLAGS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_S3 = @ENABLE_S3@ -HAVE_SODIUM = @HAVE_SODIUM@ +GTEST_LIBS = @GTEST_LIBS@ HAVE_SECCOMP = @HAVE_SECCOMP@ -BOOST_LDFLAGS = @BOOST_LDFLAGS@ +HAVE_SODIUM = @HAVE_SODIUM@ +LDFLAGS = @LDFLAGS@ +LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ +LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBLZMA_LIBS = @LIBLZMA_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_VERSION = @PACKAGE_VERSION@ SODIUM_LIBS = @SODIUM_LIBS@ -LIBLZMA_LIBS = @LIBLZMA_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ -LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ -LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ -EDITLINE_LIBS = @EDITLINE_LIBS@ bash = @bash@ bindir = @bindir@ -lsof = @lsof@ datadir = @datadir@ datarootdir = @datarootdir@ +doc_generate = @doc_generate@ docdir = @docdir@ exec_prefix = @exec_prefix@ includedir = @includedir@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ +lsof = @lsof@ mandir = @mandir@ pkglibdir = $(libdir)/$(PACKAGE_NAME) prefix = @prefix@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ -doc_generate = @doc_generate@ +system = @system@ xmllint = @xmllint@ xsltproc = @xsltproc@ @@ -1,21 +1,54 @@ +# Nix + [](https://opencollective.com/nixos) +[](https://github.com/NixOS/nix/actions) + +Nix is a powerful package manager for Linux and other Unix systems that makes package +management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +for more details. + +## Installation + +On Linux and macOS the easiest way to Install Nix is to run the following shell command +(as a user other than root): + +``` +$ curl -L https://nixos.org/nix/install | sh +``` + +Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). + +## Building And Developing + +### Building Nix + +You can build Nix using one of the targets provided by [release.nix](./release.nix): + +``` +$ nix-build ./release.nix -A build.aarch64-linux +$ nix-build ./release.nix -A build.x86_64-darwin +$ nix-build ./release.nix -A build.i686-linux +$ nix-build ./release.nix -A build.x86_64-linux +``` -Nix, the purely functional package manager ------------------------------------------- +### Development Environment -Nix is a new take on package management that is fairly unique. Because of its -purity aspects, a lot of issues found in traditional package managers don't -appear with Nix. +You can use the provided `shell.nix` to get a working development environment: -To find out more about the tool, usage and installation instructions, please -read the manual, which is available on the Nix website at -<https://nixos.org/nix/manual>. +``` +$ nix-shell +$ ./bootstrap.sh +$ ./configure +$ make +``` -## Contributing +## Additional Resources -Take a look at the [Hacking Section](https://nixos.org/nix/manual/#chap-hacking) -of the manual. It helps you to get started with building Nix from source. +- [Nix manual](https://nixos.org/nix/manual) +- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) +- [NixOS Discourse](https://discourse.nixos.org/) +- [IRC - #nixos on freenode.net](irc://irc.freenode.net/#nixos) ## License -Nix is released under the LGPL v2.1 +Nix is released under the [LGPL v2.1](./COPYING). diff --git a/configure.ac b/configure.ac index b868343d2..c3007b4b6 100644 --- a/configure.ac +++ b/configure.ac @@ -266,6 +266,10 @@ if test "$gc" = yes; then fi +# Look for gtest. +PKG_CHECK_MODULES([GTEST], [gtest_main]) + + # documentation generation switch AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], [disable documentation generation]), diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml index 08a7a772f..acfe9e3cc 100644 --- a/doc/manual/advanced-topics/post-build-hook.xml +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -61,7 +61,7 @@ substituters = https://cache.nixos.org/ s3://example-nix-cache trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= </programlisting> -<para>we will restart the Nix daemon a later step.</para> +<para>We will restart the Nix daemon in a later step.</para> </section> <section> @@ -139,7 +139,7 @@ $ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example <para>Now, copy the path back from the cache:</para> <screen> -$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +$ nix-store --realise /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... warning: you did not specify '--add-root'; the result might be removed by the garbage collector /nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 48dce7c95..1fa74a143 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -19,26 +19,30 @@ <refsection><title>Description</title> -<para>Nix reads settings from two configuration files:</para> +<para>By default Nix reads settings from the following places:</para> + +<para>The system-wide configuration file +<filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> +(i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or +<filename>$NIX_CONF_DIR/nix.conf</filename> if +<envar>NIX_CONF_DIR</envar> is set. Values loaded in this file are not forwarded to the Nix daemon. The +client assumes that the daemon has already loaded them. +</para> -<itemizedlist> +<para>User-specific configuration files:</para> - <listitem> - <para>The system-wide configuration file - <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> - (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or - <filename>$NIX_CONF_DIR/nix.conf</filename> if - <envar>NIX_CONF_DIR</envar> is set.</para> - </listitem> +<para> + If <envar>NIX_USER_CONF_FILES</envar> is set, then each path separated by + <literal>:</literal> will be loaded in reverse order. +</para> - <listitem> - <para>The user configuration file - <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or - <filename>~/.config/nix/nix.conf</filename> if - <envar>XDG_CONFIG_HOME</envar> is not set.</para> - </listitem> +<para> + Otherwise it will look for <filename>nix/nix.conf</filename> files in + <envar>XDG_CONFIG_DIRS</envar> and <envar>XDG_CONFIG_HOME</envar>. -</itemizedlist> + The default location is <filename>$HOME/.config/nix.conf</filename> if + those environment variables are unset. +</para> <para>The configuration files consist of <literal><replaceable>name</replaceable> = @@ -382,7 +386,7 @@ false</literal>.</para> <programlisting> builtins.fetchurl { - url = https://example.org/foo-1.2.3.tar.xz; + url = "https://example.org/foo-1.2.3.tar.xz"; sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; } </programlisting> diff --git a/doc/manual/command-ref/env-common.xml b/doc/manual/command-ref/env-common.xml index 696d68c34..8466cc463 100644 --- a/doc/manual/command-ref/env-common.xml +++ b/doc/manual/command-ref/env-common.xml @@ -33,7 +33,7 @@ will cause Nix to look for paths relative to <filename>/home/eelco/Dev</filename> and - <filename>/etc/nixos</filename>, in that order. It is also + <filename>/etc/nixos</filename>, in this order. It is also possible to match paths against a prefix. For example, the value <screen> @@ -53,13 +53,13 @@ nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen> <envar>NIX_PATH</envar> to <screen> -nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen> +nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-15.09.tar.gz</screen> tells Nix to download the latest revision in the Nixpkgs/NixOS 15.09 channel.</para> <para>A following shorthand can be used to refer to the official channels: - + <screen>nixpkgs=channel:nixos-15.09</screen> </para> @@ -137,12 +137,19 @@ $ mount -o bind /mnt/otherdisk/nix /nix</screen> <varlistentry><term><envar>NIX_CONF_DIR</envar></term> - <listitem><para>Overrides the location of the Nix configuration + <listitem><para>Overrides the location of the system Nix configuration directory (default <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem> </varlistentry> +<varlistentry><term><envar>NIX_USER_CONF_FILES</envar></term> + + <listitem><para>Overrides the location of the user Nix configuration files + to load from (defaults to the XDG spec locations). The variable is treated + as a list separated by the <literal>:</literal> token.</para></listitem> + +</varlistentry> <varlistentry><term><envar>TMPDIR</envar></term> diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 9c03ccce1..2b95b6819 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -526,13 +526,10 @@ these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): 14.12 channel: <screen> -$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox </screen> -(The GitHub repository <literal>nixpkgs-channels</literal> is updated -automatically from the main <literal>nixpkgs</literal> repository -after certain tests have succeeded and binaries have been built and -uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para> +</para> </refsection> diff --git a/doc/manual/command-ref/nix-shell.xml b/doc/manual/command-ref/nix-shell.xml index 766482460..2fef323c5 100644 --- a/doc/manual/command-ref/nix-shell.xml +++ b/doc/manual/command-ref/nix-shell.xml @@ -258,7 +258,7 @@ path. You can override it by passing <option>-I</option> or setting containing the Pan package from a specific revision of Nixpkgs: <screen> -$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz [nix-shell:~]$ pan --version Pan 0.139 @@ -352,7 +352,7 @@ following Haskell script uses a specific branch of Nixpkgs/NixOS (the <programlisting><![CDATA[ #! /usr/bin/env nix-shell #! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])" -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-18.03.tar.gz import Network.HTTP import Text.HTML.TagSoup @@ -370,7 +370,7 @@ If you want to be even more precise, you can specify a specific revision of Nixpkgs: <programlisting> -#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz </programlisting> </para> diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 113a3c2e4..d71f9d8e4 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -360,7 +360,6 @@ EOF <arg choice='plain'><option>--print-roots</option></arg> <arg choice='plain'><option>--print-live</option></arg> <arg choice='plain'><option>--print-dead</option></arg> - <arg choice='plain'><option>--delete</option></arg> </group> <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg> </cmdsynopsis> @@ -407,14 +406,6 @@ the Nix store not reachable via file system references from a set of </varlistentry> - <varlistentry><term><option>--delete</option></term> - - <listitem><para>This operation performs an actual garbage - collection. All dead paths are removed from the - store. This is the default.</para></listitem> - - </varlistentry> - </variablelist> <para>By default, all unreachable paths are deleted. The following @@ -444,10 +435,10 @@ and <link linkend="conf-keep-derivations"><literal>keep-derivations</literal></link> variables in the Nix configuration file.</para> -<para>With <option>--delete</option>, the collector prints the total -number of freed bytes when it finishes (or when it is interrupted). -With <option>--print-dead</option>, it prints the number of bytes that -would be freed.</para> +<para>By default, the collector prints the total number of freed bytes +when it finishes (or when it is interrupted). With +<option>--print-dead</option>, it prints the number of bytes that would +be freed.</para> </refsection> @@ -945,7 +936,7 @@ $ nix-store --add ./foo.c <para>The operation <option>--add-fixed</option> adds the specified paths to the Nix store. Unlike <option>--add</option> paths are registered using the -specified hashing algorithm, resulting in the same output path as a fixed output +specified hashing algorithm, resulting in the same output path as a fixed-output derivation. This can be used for sources that are not available from a public url or broke since the download expression was written. </para> @@ -1148,7 +1139,7 @@ the information that Nix considers important. For instance, timestamps are elided because all files in the Nix store have their timestamp set to 0 anyway. Likewise, all permissions are left out except for the execute bit, because all files in the Nix store have -644 or 755 permission.</para> +444 or 555 permission.</para> <para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning that “equal” paths always produce the same NAR archive. For instance, diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index b610b54b9..2660e3bb1 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -1,5 +1,5 @@ <nop xmlns="http://docbook.org/ns/docbook"> - + <arg><option>--help</option></arg> <arg><option>--version</option></arg> <arg rep='repeat'> @@ -12,6 +12,10 @@ <arg choice='plain'><option>--quiet</option></arg> </arg> <arg> + <option>--log-format</option> + <replaceable>format</replaceable> +</arg> +<arg> <group choice='plain'> <arg choice='plain'><option>--no-build-output</option></arg> <arg choice='plain'><option>-Q</option></arg> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index 0383bfaed..a68eef1d0 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -92,6 +92,37 @@ </varlistentry> +<varlistentry xml:id="opt-log-format"><term><option>--log-format</option> <replaceable>format</replaceable></term> + + <listitem> + + <para>This option can be used to change the output of the log format, with + <replaceable>format</replaceable> being one of:</para> + + <variablelist> + + <varlistentry><term>raw</term> + <listitem><para>This is the raw format, as outputted by nix-build.</para></listitem> + </varlistentry> + + <varlistentry><term>internal-json</term> + <listitem><para>Outputs the logs in a structured manner. NOTE: the json schema is not guarantees to be stable between releases.</para></listitem> + </varlistentry> + + <varlistentry><term>bar</term> + <listitem><para>Only display a progress bar during the builds.</para></listitem> + </varlistentry> + + <varlistentry><term>bar-with-logs</term> + <listitem><para>Display the raw logs, with the progress bar at the bottom.</para></listitem> + </varlistentry> + + </variablelist> + + </listitem> + +</varlistentry> + <varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term> <listitem><para>By default, output written by builders to standard diff --git a/doc/manual/expressions/advanced-attributes.xml b/doc/manual/expressions/advanced-attributes.xml index 372d03de7..5759ff50e 100644 --- a/doc/manual/expressions/advanced-attributes.xml +++ b/doc/manual/expressions/advanced-attributes.xml @@ -178,7 +178,7 @@ impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ]; <programlisting> fetchurl { - url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } </programlisting> @@ -189,7 +189,7 @@ fetchurl { <programlisting> fetchurl { - url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; } </programlisting> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index 394e1fc32..a18c5801a 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -324,7 +324,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> particular version of Nixpkgs, e.g. <programlisting> -with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; +with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; stdenv.mkDerivation { … } </programlisting> @@ -349,7 +349,7 @@ stdenv.mkDerivation { … } <programlisting> with import (fetchTarball { - url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz; + url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz"; sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; }) {}; @@ -422,6 +422,16 @@ stdenv.mkDerivation { … } </para> </listitem> </varlistentry> + <varlistentry> + <term>submodules</term> + <listitem> + <para> + A Boolean parameter that specifies whether submodules + should be checked out. Defaults to + <literal>false</literal>. + </para> + </listitem> + </varlistentry> </variablelist> <example> @@ -1396,7 +1406,7 @@ stdenv.mkDerivation { "; src = fetchurl { - url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; }; inherit perl; diff --git a/doc/manual/expressions/expression-syntax.xml b/doc/manual/expressions/expression-syntax.xml index 42b9dca36..a3de20713 100644 --- a/doc/manual/expressions/expression-syntax.xml +++ b/doc/manual/expressions/expression-syntax.xml @@ -15,7 +15,7 @@ stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' /> name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' /> builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' /> src = fetchurl { <co xml:id='ex-hello-nix-co-5' /> - url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + url = "ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; }; inherit perl; <co xml:id='ex-hello-nix-co-6' /> diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index 7326a3e76..ce0a1636d 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -73,12 +73,4 @@ waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</ So it is always safe to run multiple instances of Nix in parallel (which isn’t the case with, say, <command>make</command>).</para> -<para>If you have a system with multiple CPUs, you may want to have -Nix build different derivations in parallel (insofar as possible). -Just pass the option <link linkend='opt-max-jobs'><option>-j -<replaceable>N</replaceable></option></link>, where -<replaceable>N</replaceable> is the maximum number of jobs to be run -in parallel, or set. Typically this should be the number of -CPUs.</para> - </section> diff --git a/doc/manual/installation/installing-binary.xml b/doc/manual/installation/installing-binary.xml index 3f57f47b5..8d548f0ea 100644 --- a/doc/manual/installation/installing-binary.xml +++ b/doc/manual/installation/installing-binary.xml @@ -6,16 +6,30 @@ <title>Installing a Binary Distribution</title> -<para>If you are using Linux or macOS, the easiest way to install Nix -is to run the following command: +<para> + If you are using Linux or macOS versions up to 10.14 (Mojave), the + easiest way to install Nix is to run the following command: +</para> <screen> $ sh <(curl https://nixos.org/nix/install) </screen> -As of Nix 2.1.0, the Nix installer will always default to creating a -single-user installation, however opting in to the multi-user -installation is highly recommended. +<para> + If you're using macOS 10.15 (Catalina) or newer, consult + <link linkend="sect-macos-installation">the macOS installation instructions</link> + before installing. +</para> + +<para> + As of Nix 2.1.0, the Nix installer will always default to creating a + single-user installation, however opting in to the multi-user + installation is highly recommended. + <!-- TODO: this explains *neither* why the default version is + single-user, nor why we'd recommend multi-user over the default. + True prospective users don't have much basis for evaluating this. + What's it to me? Who should pick which? Why? What if I pick wrong? + --> </para> <section xml:id="sect-single-user-installation"> @@ -36,7 +50,7 @@ run this under your usual user account, <emphasis>not</emphasis> as root. The script will invoke <command>sudo</command> to create <filename>/nix</filename> if it doesn’t already exist. If you don’t have <command>sudo</command>, you should manually create -<command>/nix</command> first as root, e.g.: +<filename>/nix</filename> first as root, e.g.: <screen> $ mkdir /nix @@ -47,7 +61,7 @@ The install script will modify the first writable file from amongst <filename>.bash_profile</filename>, <filename>.bash_login</filename> and <filename>.profile</filename> to source <filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set -the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment +the <envar>NIX_INSTALLER_NO_MODIFY_PROFILE</envar> environment variable before executing the install script to disable this behaviour. </para> @@ -81,12 +95,10 @@ $ rm -rf /nix <para> You can instruct the installer to perform a multi-user installation on your system: - - <screen> - sh <(curl https://nixos.org/nix/install) --daemon -</screen> </para> + <screen>sh <(curl https://nixos.org/nix/install) --daemon</screen> + <para> The multi-user installation of Nix will create build users between the user IDs 30001 and 30032, and a group with the group ID 30000. @@ -136,6 +148,273 @@ sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist </section> +<section xml:id="sect-macos-installation"> + <title>macOS Installation</title> + + <para> + Starting with macOS 10.15 (Catalina), the root filesystem is read-only. + This means <filename>/nix</filename> can no longer live on your system + volume, and that you'll need a workaround to install Nix. + </para> + + <para> + The recommended approach, which creates an unencrypted APFS volume + for your Nix store and a "synthetic" empty directory to mount it + over at <filename>/nix</filename>, is least likely to impair Nix + or your system. + </para> + + <note><para> + With all separate-volume approaches, it's possible something on + your system (particularly daemons/services and restored apps) may + need access to your Nix store before the volume is mounted. Adding + additional encryption makes this more likely. + </para></note> + + <para> + If you're using a recent Mac with a + <link xlink:href="https://www.apple.com/euro/mac/shared/docs/Apple_T2_Security_Chip_Overview.pdf">T2 chip</link>, + your drive will still be encrypted at rest (in which case "unencrypted" + is a bit of a misnomer). To use this approach, just install Nix with: + </para> + + <screen>$ sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume</screen> + + <para> + If you don't like the sound of this, you'll want to weigh the + other approaches and tradeoffs detailed in this section. + </para> + + <note> + <title>Eventual solutions?</title> + <para> + All of the known workarounds have drawbacks, but we hope + better solutions will be available in the future. Some that + we have our eye on are: + </para> + <orderedlist> + <listitem> + <para> + A true firmlink would enable the Nix store to live on the + primary data volume without the build problems caused by + the symlink approach. End users cannot currently + create true firmlinks. + </para> + </listitem> + <listitem> + <para> + If the Nix store volume shared FileVault encryption + with the primary data volume (probably by using the same + volume group and role), FileVault encryption could be + easily supported by the installer without requiring + manual setup by each user. + </para> + </listitem> + </orderedlist> + </note> + + <section xml:id="sect-macos-installation-change-store-prefix"> + <title>Change the Nix store path prefix</title> + <para> + Changing the default prefix for the Nix store is a simple + approach which enables you to leave it on your root volume, + where it can take full advantage of FileVault encryption if + enabled. Unfortunately, this approach also opts your device out + of some benefits that are enabled by using the same prefix + across systems: + + <itemizedlist> + <listitem> + <para> + Your system won't be able to take advantage of the binary + cache (unless someone is able to stand up and support + duplicate caching infrastructure), which means you'll + spend more time waiting for builds. + </para> + </listitem> + <listitem> + <para> + It's harder to build and deploy packages to Linux systems. + </para> + </listitem> + <!-- TODO: may be more here --> + </itemizedlist> + + <!-- TODO: Yes, but how?! --> + + It would also possible (and often requested) to just apply this + change ecosystem-wide, but it's an intrusive process that has + side effects we want to avoid for now. + <!-- magnificent hand-wavy gesture --> + </para> + <para> + </para> + </section> + + <section xml:id="sect-macos-installation-encrypted-volume"> + <title>Use a separate encrypted volume</title> + <para> + If you like, you can also add encryption to the recommended + approach taken by the installer. You can do this by pre-creating + an encrypted volume before you run the installer--or you can + run the installer and encrypt the volume it creates later. + <!-- TODO: see later note about whether this needs both add-encryption and from-scratch directions --> + </para> + <para> + In either case, adding encryption to a second volume isn't quite + as simple as enabling FileVault for your boot volume. Before you + dive in, there are a few things to weigh: + </para> + <orderedlist> + <listitem> + <para> + The additional volume won't be encrypted with your existing + FileVault key, so you'll need another mechanism to decrypt + the volume. + </para> + </listitem> + <listitem> + <para> + You can store the password in Keychain to automatically + decrypt the volume on boot--but it'll have to wait on Keychain + and may not mount before your GUI apps restore. If any of + your launchd agents or apps depend on Nix-installed software + (for example, if you use a Nix-installed login shell), the + restore may fail or break. + </para> + <para> + On a case-by-case basis, you may be able to work around this + problem by using <command>wait4path</command> to block + execution until your executable is available. + </para> + <para> + It's also possible to decrypt and mount the volume earlier + with a login hook--but this mechanism appears to be + deprecated and its future is unclear. + </para> + </listitem> + <listitem> + <para> + You can hard-code the password in the clear, so that your + store volume can be decrypted before Keychain is available. + </para> + </listitem> + </orderedlist> + <para> + If you are comfortable navigating these tradeoffs, you can encrypt the volume with + something along the lines of: + <!-- TODO: + I don't know if this also needs from-scratch instructions? + can we just recommend use-the-installer-and-then-encrypt? + --> + </para> + <!-- + TODO: it looks like this option can be encryptVolume|encrypt|enableFileVault + + It may be more clear to use encryptVolume, here? FileVault seems + heavily associated with the boot-volume behavior; I worry + a little that it can mislead here, especially as it gets + copied around minus doc context...? + --> + <screen>alice$ diskutil apfs enableFileVault /nix -user disk</screen> + + <!-- TODO: and then go into detail on the mount/decrypt approaches? --> + </section> + + <section xml:id="sect-macos-installation-symlink"> + <!-- + Maybe a good razor is: if we'd hate having to support someone who + installed Nix this way, it shouldn't even be detailed? + --> + <title>Symlink the Nix store to a custom location</title> + <para> + Another simple approach is using <filename>/etc/synthetic.conf</filename> + to symlink the Nix store to the data volume. This option also + enables your store to share any configured FileVault encryption. + Unfortunately, builds that resolve the symlink may leak the + canonical path or even fail. + </para> + <para> + Because of these downsides, we can't recommend this approach. + </para> + <!-- Leaving out instructions for this one. --> + </section> + + <section xml:id="sect-macos-installation-recommended-notes"> + <title>Notes on the recommended approach</title> + <para> + This section goes into a little more detail on the recommended + approach. You don't need to understand it to run the installer, + but it can serve as a helpful reference if you run into trouble. + </para> + <orderedlist> + <listitem> + <para> + In order to compose user-writable locations into the new + read-only system root, Apple introduced a new concept called + <literal>firmlinks</literal>, which it describes as a + "bi-directional wormhole" between two filesystems. You can + see the current firmlinks in <filename>/usr/share/firmlinks</filename>. + Unfortunately, firmlinks aren't (currently?) user-configurable. + </para> + + <para> + For special cases like NFS mount points or package manager roots, + <link xlink:href="https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man5/synthetic.conf.5.html">synthetic.conf(5)</link> + supports limited user-controlled file-creation (of symlinks, + and synthetic empty directories) at <filename>/</filename>. + To create a synthetic empty directory for mounting at <filename>/nix</filename>, + add the following line to <filename>/etc/synthetic.conf</filename> + (create it if necessary): + </para> + + <screen>nix</screen> + </listitem> + + <listitem> + <para> + This configuration is applied at boot time, but you can use + <command>apfs.util</command> to trigger creation (not deletion) + of new entries without a reboot: + </para> + + <screen>alice$ /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B</screen> + </listitem> + + <listitem> + <para> + Create the new APFS volume with diskutil: + </para> + + <screen>alice$ sudo diskutil apfs addVolume diskX APFS 'Nix Store' -mountpoint /nix</screen> + </listitem> + + <listitem> + <para> + Using <command>vifs</command>, add the new mount to + <filename>/etc/fstab</filename>. If it doesn't already have + other entries, it should look something like: + </para> + +<screen> +# +# Warning - this file should only be modified with vifs(8) +# +# Failure to do so is unsupported and may be destructive. +# +LABEL=Nix\040Store /nix apfs rw,nobrowse +</screen> + + <para> + The nobrowse setting will keep Spotlight from indexing this + volume, and keep it from showing up on your desktop. + </para> + </listitem> + </orderedlist> + </section> + +</section> + <section xml:id="sect-nix-install-pinned-version-url"> <title>Installing a pinned Nix version from a URL</title> diff --git a/doc/manual/installation/upgrading.xml b/doc/manual/installation/upgrading.xml index 30670d7fe..592f63895 100644 --- a/doc/manual/installation/upgrading.xml +++ b/doc/manual/installation/upgrading.xml @@ -17,6 +17,11 @@ <para> Single-user installations of Nix should run this: - <command>nix-channel --update; nix-env -iA nixpkgs.nix</command> + <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert</command> + </para> + + <para> + Multi-user Nix users on Linux should run this with sudo: + <command>nix-channel --update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon</command> </para> </chapter> diff --git a/doc/manual/release-notes/rl-0.8.xml b/doc/manual/release-notes/rl-0.8.xml index 784b26c6b..825798fa9 100644 --- a/doc/manual/release-notes/rl-0.8.xml +++ b/doc/manual/release-notes/rl-0.8.xml @@ -8,7 +8,7 @@ <para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). As a result, <command>nix-pull</command> manifests and channels built -for Nix 0.7 and below will now work anymore. However, the Nix +for Nix 0.7 and below will not work anymore. However, the Nix expression language has not changed, so you can still build from source. Also, existing user environments continue to work. Nix 0.8 will automatically upgrade the database schema of previous @@ -6,9 +6,11 @@ dist-files += configure config.h.in perl/configure clean-files += Makefile.config -GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix -Wno-deprecated-declarations +GLOBAL_CXXFLAGS += -Wno-deprecated-declarations $(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) $(GCH) $(PCH): src/libutil/util.hh config.h + +GCH_CXXFLAGS = -I src/libutil diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77534babb..baefe0f12 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -1,5 +1,5 @@ #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1 +#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp perlPackages.NetAmazonS3 gnupg1 use strict; use Data::Dumper; @@ -9,12 +9,16 @@ use File::Slurp; use File::Copy; use JSON::PP; use LWP::UserAgent; +use Net::Amazon::S3; my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; -my $releasesDir = "/home/eelco/mnt/releases"; +my $releasesBucketName = "nix-releases"; +my $channelsBucketName = "nix-channels"; my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; +my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; + # FIXME: cut&paste from nixos-channel-scripts. sub fetch { my ($url, $type) = @_; @@ -42,13 +46,31 @@ my $version = $1; print STDERR "Nix revision is $nixRev, version is $version\n"; -File::Path::make_path($releasesDir); -if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; -} +my $releaseDir = "nix/$releaseName"; + +my $tmpDir = "$TMPDIR/nix-release/$releaseName"; +File::Path::make_path($tmpDir); + +# S3 setup. +my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "No AWS_ACCESS_KEY_ID given."; +my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "No AWS_SECRET_ACCESS_KEY given."; -my $releaseDir = "$releasesDir/nix/$releaseName"; -File::Path::make_path($releaseDir); +my $s3 = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + host => "s3-eu-west-1.amazonaws.com", + }); + +my $releasesBucket = $s3->bucket($releasesBucketName) or die; + +my $s3_us = Net::Amazon::S3->new( + { aws_access_key_id => $aws_access_key_id, + aws_secret_access_key => $aws_secret_access_key, + retry => 1, + }); + +my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub downloadFile { my ($jobName, $productNr, $dstName) = @_; @@ -57,40 +79,49 @@ sub downloadFile { my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n"; $dstName //= basename($srcFile); - my $dstFile = "$releaseDir/" . $dstName; + my $tmpFile = "$tmpDir/$dstName"; - if (! -e $dstFile) { - print STDERR "downloading $srcFile to $dstFile...\n"; - system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + if (!-e $tmpFile) { + print STDERR "downloading $srcFile to $tmpFile...\n"; + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$tmpFile'") == 0 or die "unable to fetch $srcFile\n"; - rename("$dstFile.tmp", $dstFile) or die; } my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; - my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$tmpFile'`; chomp $sha256_actual; if ($sha256_expected ne $sha256_actual) { - print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; + print STDERR "file $tmpFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; exit 1; } - write_file("$dstFile.sha256", $sha256_expected); + write_file("$tmpFile.sha256", $sha256_expected); - if (! -e "$dstFile.asc") { - system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n"; + if (! -e "$tmpFile.asc") { + system("gpg2 --detach-sign --armor $tmpFile") == 0 or die "unable to sign $tmpFile\n"; } - return ($dstFile, $sha256_expected); + return $sha256_expected; } downloadFile("tarball", "2"); # .tar.bz2 -my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz +my $tarballHash = downloadFile("tarball", "3"); # .tar.xz downloadFile("binaryTarball.i686-linux", "1"); downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("installerScript", "1"); +for my $fn (glob "$tmpDir/*") { + my $name = basename($fn); + my $dstKey = "$releaseDir/" . $name; + unless (defined $releasesBucket->head_key($dstKey)) { + print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; + $releasesBucket->add_key_filename($dstKey, $fn) + or die $releasesBucket->err . ": " . $releasesBucket->errstr; + } +} + exit if $version =~ /pre/; # Update Nixpkgs in a very hacky way. @@ -111,8 +142,12 @@ $oldName =~ s/"//g; sub getStorePath { my ($jobName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build"; - return $buildInfo->{buildproducts}->{1}->{path}; + for my $product (values %{$buildInfo->{buildproducts}}) { + next unless $product->{type} eq "nix-build"; + next if $product->{path} =~ /[a-z]+$/; + return $product->{path}; + } + die; } write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", @@ -125,18 +160,11 @@ write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; -# Extract the HTML manual. -File::Path::make_path("$releaseDir/manual"); - -system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die; - -if (! -e "$releaseDir/manual/index.html") { - symlink("manual.html", "$releaseDir/manual/index.html") or die; -} - # Update the "latest" symlink. -symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die; -rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die; +$channelsBucket->add_key( + "nix-latest/install", "", + { "x-amz-website-redirect-location" => "https://releases.nixos.org/$releaseDir/install" }) + or die $channelsBucket->err . ": " . $channelsBucket->errstr; # Tag the release in Git. chdir("/home/eelco/Dev/nix-pristine") or die; diff --git a/mk/libraries.mk b/mk/libraries.mk index 307e29b9d..e6ef2e3ec 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -125,7 +125,8 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).a $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ - $(trace-ar) $(AR) crs $$@ $$? + $(trace-ld) $(LD) -Ur -o $$(_d)/$$($(1)_NAME).o $$? + $(trace-ar) $(AR) crs $$@ $$(_d)/$$($(1)_NAME).o $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index 1a727ba1b..1c0452dc2 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,14 +8,14 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) PCH = $(buildprefix)precompiled-headers.h.pch $(PCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) + $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) $(PCH) diff --git a/mk/programs.mk b/mk/programs.mk index d93df4468..3fa9685c3 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -35,24 +35,28 @@ define build-program $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) - $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) + ifdef $(1)_INSTALL_DIR - install: $(DESTDIR)$$($(1)_INSTALL_PATH) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) - ifeq ($(BUILD_SHARED_LIBS), 1) + $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) - _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + install: $(DESTDIR)$$($(1)_INSTALL_PATH) - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + ifeq ($(BUILD_SHARED_LIBS), 1) + + _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) - else + else - $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + endif endif # Propagate CFLAGS and CXXFLAGS to the individual object files. @@ -76,4 +80,10 @@ define build-program programs-list += $$($(1)_PATH) clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) dist-files += $$(_srcs) + + # Phony target to run this program (typically as a dependency of 'check'). + .PHONY: $(1)_RUN + $(1)_RUN: $$($(1)_PATH) + $(trace-test) $$($(1)_PATH) + endef diff --git a/mk/tracing.mk b/mk/tracing.mk index 13912d3c7..54c77ab60 100644 --- a/mk/tracing.mk +++ b/mk/tracing.mk @@ -11,6 +11,7 @@ ifeq ($(V), 0) trace-javac = @echo " JAVAC " $@; trace-jar = @echo " JAR " $@; trace-mkdir = @echo " MKDIR " $@; + trace-test = @echo " TEST " $@; suppress = @ diff --git a/nix-rust/local.mk b/nix-rust/local.mk index 1e006e500..e4bfde31b 100644 --- a/nix-rust/local.mk +++ b/nix-rust/local.mk @@ -41,5 +41,5 @@ ifneq ($(OS), Darwin) check: rust-tests rust-tests: - cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) + $(trace-test) cd nix-rust && CARGO_HOME=$$(if [[ -d vendor ]]; then echo vendor; fi) cargo test --release $$(if [[ -d vendor ]]; then echo --offline; fi) endif diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 1ca734e75..945ed49c7 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -80,7 +80,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16); + auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -274,7 +274,8 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, recursive, parseHashType(algo)); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -285,7 +286,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { Hash h(hash, parseHashType(algo)); - auto path = store()->makeFixedOutputPath(recursive, h, name); + auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto path = store()->makeFixedOutputPath(method, h, name); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/precompiled-headers.h b/precompiled-headers.h index e0d885b23..079aa496e 100644 --- a/precompiled-headers.h +++ b/precompiled-headers.h @@ -56,6 +56,3 @@ #include <sys/wait.h> #include <termios.h> #include <unistd.h> - -#include "util.hh" -#include "args.hh" diff --git a/release-common.nix b/release-common.nix index b6c8f3d81..4316c3c23 100644 --- a/release-common.nix +++ b/release-common.nix @@ -50,11 +50,11 @@ rec { libarchive boost nlohmann_json - rustc cargo # Tests git mercurial + gmock ] ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium diff --git a/release.nix b/release.nix index 9b591229a..fbf9e4721 100644 --- a/release.nix +++ b/release.nix @@ -14,50 +14,6 @@ let jobs = rec { - # Create a "vendor" directory that contains the crates listed in - # Cargo.lock. This allows Nix to be built without network access. - vendoredCrates = - let - lockFile = builtins.fromTOML (builtins.readFile nix-rust/Cargo.lock); - - files = map (pkg: import <nix/fetchurl.nix> { - url = "https://crates.io/api/v1/crates/${pkg.name}/${pkg.version}/download"; - sha256 = lockFile.metadata."checksum ${pkg.name} ${pkg.version} (registry+https://github.com/rust-lang/crates.io-index)"; - }) (builtins.filter (pkg: pkg.source or "" == "registry+https://github.com/rust-lang/crates.io-index") lockFile.package); - - in pkgs.runCommand "cargo-vendor-dir" {} - '' - mkdir -p $out/vendor - - cat > $out/vendor/config <<EOF - [source.crates-io] - replace-with = "vendored-sources" - - [source.vendored-sources] - directory = "vendor" - EOF - - ${toString (builtins.map (file: '' - mkdir $out/vendor/tmp - tar xvf ${file} -C $out/vendor/tmp - dir=$(echo $out/vendor/tmp/*) - - # Add just enough metadata to keep Cargo happy. - printf '{"files":{},"package":"${file.outputHash}"}' > "$dir/.cargo-checksum.json" - - # Clean up some cruft from the winapi crates. FIXME: find - # a way to remove winapi* from our dependencies. - if [[ $dir =~ /winapi ]]; then - find $dir -name "*.a" -print0 | xargs -0 rm -f -- - fi - - mv "$dir" $out/vendor/ - - rm -rf $out/vendor/tmp - '') files)} - ''; - - build = pkgs.lib.genAttrs systems (system: let pkgs = import nixpkgs { inherit system; }; in @@ -89,8 +45,6 @@ let patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} - ln -sfn ${vendoredCrates}/vendor/ nix-rust/vendor - (cd perl; autoreconf --install --force --verbose) ''; @@ -103,17 +57,17 @@ let installFlags = "sysconfdir=$(out)/etc"; + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ''; + doCheck = true; doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; separateDebugInfo = true; - - preDist = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ''; }); @@ -165,10 +119,10 @@ let } '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ --subst-var-by nix ${toplevel} \ --subst-var-by cacert ${cacert} @@ -183,6 +137,7 @@ let # SC1090: Don't worry about not being able to find # $nix/etc/profile.d/nix.sh shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh shellcheck $TMPDIR/install-darwin-multi-user.sh shellcheck $TMPDIR/install-systemd-multi-user.sh @@ -198,6 +153,7 @@ let fi chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh chmod +x $TMPDIR/install-darwin-multi-user.sh chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user @@ -210,11 +166,15 @@ let --absolute-names \ --hard-dereference \ --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ $(cat ${installerClosureInfo}/store-paths) ''); diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh new file mode 100755 index 000000000..dac30d72d --- /dev/null +++ b/scripts/create-darwin-volume.sh @@ -0,0 +1,185 @@ +#!/bin/sh +set -e + +root_disk() { + diskutil info -plist / +} + +apfs_volumes_for() { + disk=$1 + diskutil apfs list -plist "$disk" +} + +disk_identifier() { + xpath "/plist/dict/key[text()='ParentWholeDisk']/following-sibling::string[1]/text()" 2>/dev/null +} + +volume_list_true() { + key=$1 + xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict/key[text()='$key']/following-sibling::true[1]" 2> /dev/null +} + +volume_get_string() { + key=$1 i=$2 + xpath "/plist/dict/array/dict/key[text()='Volumes']/following-sibling::array/dict[$i]/key[text()='$key']/following-sibling::string[1]/text()" 2> /dev/null +} + +find_nix_volume() { + disk=$1 + i=1 + volumes=$(apfs_volumes_for "$disk") + while true; do + name=$(echo "$volumes" | volume_get_string "Name" "$i") + if [ -z "$name" ]; then + break + fi + case "$name" in + [Nn]ix*) + echo "$name" + break + ;; + esac + i=$((i+1)) + done +} + +test_fstab() { + grep -q "/nix apfs rw" /etc/fstab 2>/dev/null +} + +test_nix_symlink() { + [ -L "/nix" ] || grep -q "^nix." /etc/synthetic.conf 2>/dev/null +} + +test_synthetic_conf() { + grep -q "^nix$" /etc/synthetic.conf 2>/dev/null +} + +test_nix() { + test -d "/nix" +} + +test_t2_chip_present(){ + # Use xartutil to see if system has a t2 chip. + # + # This isn't well-documented on its own; until it is, + # let's keep track of knowledge/assumptions. + # + # Warnings: + # - Don't search "xart" if porn will cause you trouble :) + # - Other xartutil flags do dangerous things. Don't run them + # naively. If you must, search "xartutil" first. + # + # Assumptions: + # - the "xART session seeds recovery utility" + # appears to interact with xartstorageremoted + # - `sudo xartutil --list` lists xART sessions + # and their seeds and exits 0 if successful. If + # not, it exits 1 and prints an error such as: + # xartutil: ERROR: No supported link to the SEP present + # - xART sessions/seeds are present when a T2 chip is + # (and not, otherwise) + # - the presence of a T2 chip means a newly-created + # volume on the primary drive will be + # encrypted at rest + # - all together: `sudo xartutil --list` + # should exit 0 if a new Nix Store volume will + # be encrypted at rest, and exit 1 if not. + sudo xartutil --list >/dev/null 2>/dev/null +} + +test_filevault_in_use() { + disk=$1 + # list vols on disk | get value of Filevault key | value is true + apfs_volumes_for "$disk" | volume_list_true FileVault | grep -q true +} + +# use after error msg for conditions we don't understand +suggest_report_error(){ + # ex "error: something sad happened :(" >&2 + echo " please report this @ https://github.com/nixos/nix/issues" >&2 +} + +main() { + ( + echo "" + echo " ------------------------------------------------------------------ " + echo " | This installer will create a volume for the nix store and |" + echo " | configure it to mount at /nix. Follow these steps to uninstall. |" + echo " ------------------------------------------------------------------ " + echo "" + echo " 1. Remove the entry from fstab using 'sudo vifs'" + echo " 2. Destroy the data volume using 'diskutil apfs deleteVolume'" + echo " 3. Remove the 'nix' line from /etc/synthetic.conf or the file" + echo "" + ) >&2 + + if test_nix_symlink; then + echo "error: /nix is a symlink, please remove it and make sure it's not in synthetic.conf (in which case a reboot is required)" >&2 + echo " /nix -> $(readlink "/nix")" >&2 + exit 2 + fi + + if ! test_synthetic_conf; then + echo "Configuring /etc/synthetic.conf..." >&2 + echo nix | sudo tee -a /etc/synthetic.conf + if ! test_synthetic_conf; then + echo "error: failed to configure synthetic.conf;" >&2 + suggest_report_error + exit 1 + fi + fi + + if ! test_nix; then + echo "Creating mountpoint for /nix..." >&2 + /System/Library/Filesystems/apfs.fs/Contents/Resources/apfs.util -B || true + if ! test_nix; then + sudo mkdir -p /nix 2>/dev/null || true + fi + if ! test_nix; then + echo "error: failed to bootstrap /nix; if a reboot doesn't help," >&2 + suggest_report_error + exit 1 + fi + fi + + disk=$(root_disk | disk_identifier) + volume=$(find_nix_volume "$disk") + if [ -z "$volume" ]; then + echo "Creating a Nix Store volume..." >&2 + + if test_filevault_in_use "$disk"; then + # TODO: Not sure if it's in-scope now, but `diskutil apfs list` + # shows both filevault and encrypted at rest status, and it + # may be the more semantic way to test for this? It'll show + # `FileVault: No (Encrypted at rest)` + # `FileVault: No` + # `FileVault: Yes (Unlocked)` + # and so on. + if test_t2_chip_present; then + echo "warning: boot volume is FileVault-encrypted, but the Nix store volume" >&2 + echo " is only encrypted at rest." >&2 + echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 + else + echo "error: refusing to create Nix store volume because the boot volume is" >&2 + echo " FileVault encrypted, but encryption-at-rest is not available." >&2 + echo " Manually create a volume for the store and re-run this script." >&2 + echo " See https://nixos.org/nix/manual/#sect-macos-installation" >&2 + exit 1 + fi + fi + + sudo diskutil apfs addVolume "$disk" APFS 'Nix Store' -mountpoint /nix + volume="Nix Store" + else + echo "Using existing '$volume' volume" >&2 + fi + + if ! test_fstab; then + echo "Configuring /etc/fstab..." >&2 + label=$(echo "$volume" | sed 's/ /\\040/g') + printf "\$a\nLABEL=%s /nix apfs rw,nobrowse\n.\nwq\n" "$label" | EDITOR=ed sudo vifs + fi +} + +main "$@" diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 13762cba3..157e8ddb4 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -13,22 +13,25 @@ set -o pipefail # however tracking which bits came from which would be impossible. readonly ESC='\033[0m' -readonly BOLD='\033[38;1m' -readonly BLUE='\033[38;34m' -readonly BLUE_UL='\033[38;4;34m' -readonly GREEN='\033[38;32m' -readonly GREEN_UL='\033[38;4;32m' -readonly RED='\033[38;31m' - -readonly NIX_USER_COUNT="32" +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' + +# installer allows overriding build user count to speed up installation +# as creating each user takes non-trivial amount of time on macos +readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} readonly NIX_BUILD_GROUP_ID="30000" readonly NIX_BUILD_GROUP_NAME="nixbld" readonly NIX_FIRST_BUILD_UID="30001" # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. readonly NIX_ROOT="/nix" +readonly NIX_EXTRA_CONF=${NIX_EXTRA_CONF:-} -readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshenv") readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" @@ -450,9 +453,11 @@ create_directories() { } place_channel_configuration() { - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" - _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + fi } welcome_to_nix() { @@ -567,7 +572,7 @@ install_from_extracted_nix() { cd "$EXTRACTED_NIX_PATH" _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ - rsync -rlpt ./store/* "$NIX_ROOT/store/" + rsync -rlpt --chmod=-w ./store/* "$NIX_ROOT/store/" if [ -d "$NIX_INSTALLED_NIX" ]; then echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" @@ -634,18 +639,20 @@ setup_default_profile() { export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt fi - # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, - # otherwise it will be lost in environments where sudo doesn't pass - # all the environment variables by default. - _sudo "to update the default channel in the default profile" \ - HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ - || channel_update_failed=1 - + if [ -z "${NIX_INSTALLER_NO_CHANNEL_ADD:-}" ]; then + # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, + # otherwise it will be lost in environments where sudo doesn't pass + # all the environment variables by default. + _sudo "to update the default channel in the default profile" \ + HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ + || channel_update_failed=1 + fi } place_nix_configuration() { cat <<EOF > "$SCRATCH/nix.conf" +$NIX_EXTRA_CONF build-users-group = $NIX_BUILD_GROUP_NAME EOF _sudo "to place the default nix daemon configuration (part 2)" \ diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index e00708f6c..826ca8b8c 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -40,29 +40,85 @@ elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then fi INSTALL_MODE=no-daemon -# Trivially handle the --daemon / --no-daemon options -if [ "x${1:-}" = "x--no-daemon" ]; then - INSTALL_MODE=no-daemon -elif [ "x${1:-}" = "x--daemon" ]; then - INSTALL_MODE=daemon -elif [ "x${1:-}" != "x" ]; then - ( - echo "Nix Installer [--daemon|--no-daemon]" - - echo "Choose installation method." - echo "" - echo " --daemon: Installs and configures a background daemon that manages the store," - echo " providing multi-user support and better isolation for local builds." - echo " Both for security and reproducibility, this method is recommended if" - echo " supported on your platform." - echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" - echo "" - echo " --no-daemon: Simple, single-user installation that does not require root and is" - echo " trivial to uninstall." - echo " (default)" - echo "" - ) >&2 - exit +CREATE_DARWIN_VOLUME=0 +# handle the command line flags +while [ $# -gt 0 ]; do + case $1 in + --daemon) + INSTALL_MODE=daemon;; + --no-daemon) + INSTALL_MODE=no-daemon;; + --no-channel-add) + export NIX_INSTALLER_NO_CHANNEL_ADD=1;; + --daemon-user-count) + export NIX_USER_COUNT=$2 + shift;; + --no-modify-profile) + NIX_INSTALLER_NO_MODIFY_PROFILE=1;; + --darwin-use-unencrypted-nix-store-volume) + CREATE_DARWIN_VOLUME=1;; + --nix-extra-conf-file) + export NIX_EXTRA_CONF="$(cat $2)" + shift;; + *) + ( + echo "Nix Installer [--daemon|--no-daemon] [--daemon-user-count INT] [--no-channel-add] [--no-modify-profile] [--darwin-use-unencrypted-nix-store-volume] [--nix-extra-conf-file FILE]" + + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + echo " --no-channel-add: Don't add any channels. nixpkgs-unstable is installed by default." + echo "" + echo " --no-modify-profile: Skip channel installation. When not provided nixpkgs-unstable" + echo " is installed by default." + echo "" + echo " --daemon-user-count: Number of build users to create. Defaults to 32." + echo "" + echo " --nix-extra-conf-file: Path to nix.conf to prepend when installing /etc/nix.conf" + echo "" + ) >&2 + + # darwin and Catalina+ + if [ "$(uname -s)" = "Darwin" ] && [ "$macos_major" -gt 14 ]; then + ( + echo " --darwin-use-unencrypted-nix-store-volume: Create an APFS volume for the Nix" + echo " store and mount it at /nix. This is the recommended way to create" + echo " /nix with a read-only / on macOS >=10.15." + echo " See: https://nixos.org/nix/manual/#sect-macos-installation" + echo "" + ) >&2 + fi + exit;; + esac + shift +done + +if [ "$(uname -s)" = "Darwin" ]; then + if [ "$CREATE_DARWIN_VOLUME" = 1 ]; then + printf '\e[1;31mCreating volume and mountpoint /nix.\e[0m\n' + "$self/create-darwin-volume.sh" + fi + + info=$(diskutil info -plist / | xpath "/plist/dict/key[text()='Writable']/following-sibling::true[1]" 2> /dev/null) + if ! [ -e $dest ] && [ -n "$info" ] && [ "$macos_major" -gt 14 ]; then + ( + echo "" + echo "Installing on macOS >=10.15 requires relocating the store to an apfs volume." + echo "Use sh <(curl https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume or run the preparation steps manually." + echo "See https://nixos.org/nix/manual/#sect-macos-installation" + echo "" + ) >&2 + exit 1 + fi fi if [ "$INSTALL_MODE" = "daemon" ]; then @@ -130,13 +186,15 @@ if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then fi # Subscribe the user to the Nixpkgs channel and fetch it. -if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable -fi -if [ -z "$_NIX_INSTALLER_TEST" ]; then - if ! $nix/bin/nix-channel --update nixpkgs; then - echo "Fetching the nixpkgs channel failed. (Are you offline?)" - echo "To try again later, run \"nix-channel --update nixpkgs\"." +if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then + if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then + $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + fi + if [ -z "$_NIX_INSTALLER_TEST" ]; then + if ! $nix/bin/nix-channel --update nixpkgs; then + echo "Fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"nix-channel --update nixpkgs\"." + fi fi fi @@ -155,6 +213,17 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then break fi done + for i in .zshenv .zshrc; do + fn="$HOME/$i" + if [ -w "$fn" ]; then + if ! grep -q "$p" "$fn"; then + echo "modifying $fn..." >&2 + echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + fi + added=1 + break + fi + done fi if [ -z "$added" ]; then diff --git a/scripts/install.in b/scripts/install.in index 4500fd54a..1d26c4ff0 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -36,6 +36,9 @@ tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" require_util tar "unpack the binary tarball" +if [ "$(uname -s)" != "Darwin" ]; then + require_util xz "unpack the binary tarball" +fi echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 69d1c6f7e..e07117496 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,7 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; using std::cin; @@ -200,9 +200,12 @@ static int _main(int argc, char * * argv) } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); - printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), - (msg.empty() ? "" : ": " + msg)); + logError({ + .name = "Remote build", + .hint = hintfmt("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + (msg.empty() ? "" : ": " + msg)) + }); bestMachine->enabled = false; continue; } @@ -241,7 +244,7 @@ connected: uploadLock = -1; - BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string()))); + auto drv = store->readDerivation(*drvPath); drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 06b472d8b..8980bc09d 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(const string & s) ++i; while (1) { if (i == s.end()) - throw Error(format("missing closing quote in selection path '%1%'") % s); + throw Error("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -32,15 +32,13 @@ static Strings parseAttrPath(const string & s) } -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); - Error attrError = - Error(format("attribute selection path '%1%' does not match expression") % attrPath); - Value * v = &vIn; + Pos pos = noPos; for (auto & attr : tokens) { @@ -62,34 +60,36 @@ Value * findAlongAttrPath(EvalState & state, const string & attrPath, if (v->type != tAttrs) throw TypeError( - format("the expression selected by the selection path '%1%' should be a set but is %2%") - % attrPath % showType(*v)); - + "the expression selected by the selection path '%1%' should be a set but is %2%", + attrPath, + showType(*v)); if (attr.empty()) - throw Error(format("empty attribute name in selection path '%1%'") % attrPath); + throw Error("empty attribute name in selection path '%1%'", attrPath); Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); if (a == v->attrs->end()) - throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); + throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); v = &*a->value; + pos = *a->pos; } else if (apType == apIndex) { if (!v->isList()) throw TypeError( - format("the expression selected by the selection path '%1%' should be a list but is %2%") - % attrPath % showType(*v)); - + "the expression selected by the selection path '%1%' should be a list but is %2%", + attrPath, + showType(*v)); if (attrIndex >= v->listSize()) - throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); v = v->listElems()[attrIndex]; + pos = noPos; } } - return v; + return {v, pos}; } @@ -98,9 +98,9 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) Value * v2; try { auto dummyArgs = state.allocBindings(0); - v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v); + v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; } catch (Error &) { - throw Error("package '%s' has no source location information", what); + throw NoPositionInfo("package '%s' has no source location information", what); } // FIXME: is it possible to extract the Pos object instead of doing this diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 716e5ba27..fce160da7 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -7,7 +7,10 @@ namespace nix { -Value * findAlongAttrPath(EvalState & state, const string & attrPath, +MakeError(AttrPathNotFound, Error); +MakeError(NoPositionInfo, Error); + +std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 0785897d2..b1d61a285 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -43,6 +43,12 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } +Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +{ + return allocAttr(vAttrs, symbols.create(name)); +} + + void Bindings::sort() { std::sort(begin(), end()); diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 118c7bd5d..c601d09c2 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -76,7 +76,11 @@ public: { auto a = get(name); if (!a) - throw Error("attribute '%s' missing, at %s", name, pos); + throw Error({ + .hint = hintfmt("attribute '%s' missing", name), + .nixCode = NixCode { .errPos = pos } + }); + return *a; } diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 13950ab8d..44baadd53 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -1,31 +1,36 @@ #include "common-eval-args.hh" #include "shared.hh" -#include "download.hh" +#include "filetransfer.hh" #include "util.hh" #include "eval.hh" +#include "fetchers.hh" +#include "store-api.hh" namespace nix { MixEvalArgs::MixEvalArgs() { - mkFlag() - .longName("arg") - .description("argument to be passed to Nix functions") - .labels({"name", "expr"}) - .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); + addFlag({ + .longName = "arg", + .description = "argument to be passed to Nix functions", + .labels = {"name", "expr"}, + .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + }); - mkFlag() - .longName("argstr") - .description("string-valued argument to be passed to Nix functions") - .labels({"name", "string"}) - .handler([&](std::vector<std::string> ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); + addFlag({ + .longName = "argstr", + .description = "string-valued argument to be passed to Nix functions", + .labels = {"name", "string"}, + .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + }); - mkFlag() - .shortName('I') - .longName("include") - .description("add a path to the list of locations used to look up <...> file names") - .label("path") - .handler([&](std::string s) { searchPath.push_back(s); }); + addFlag({ + .longName = "include", + .shortName = 'I', + .description = "add a path to the list of locations used to look up <...> file names", + .labels = {"path"}, + .handler = {[&](std::string s) { searchPath.push_back(s); }} + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) @@ -46,9 +51,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) { - CachedDownloadRequest request(s); - request.unpack = true; - return getDownloader()->downloadCached(state.store, request).path; + return state.store->toRealPath( + fetchers::downloadTarball( + state.store, resolveUri(s), "source", false).storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index c27116e3b..3d544c903 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -7,20 +7,26 @@ namespace nix { -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { - throw EvalError(format(s) % pos); + throw EvalError({ + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) { - throw TypeError(format(s) % showType(v)); + throw TypeError(s, showType(v)); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { - throw TypeError(format(s) % showType(v) % pos); + throw TypeError({ + .hint = hintfmt(s, showType(v)), + .nixCode = NixCode { .errPos = pos } + }); } @@ -43,7 +49,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) else if (v.type == tApp) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.type == tBlackhole) - throwEvalError("infinite recursion encountered, at %1%", pos); + throwEvalError(pos, "infinite recursion encountered"); } @@ -57,9 +63,9 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tAttrs) - throwTypeError("value is %1% while a set was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a set was expected", v); } @@ -73,9 +79,9 @@ inline void EvalState::forceList(Value & v) inline void EvalState::forceList(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (!v.isList()) - throwTypeError("value is %1% while a list was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a list was expected", v); } /* Note: Various places expect the allocated memory to be zeroed. */ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index dac32b6f5..8e71db2b8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,7 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" -#include "download.hh" +#include "filetransfer.hh" #include "json.hh" #include "function-trace.hh" @@ -22,6 +22,8 @@ #if HAVE_BOEHMGC +#define GC_INCLUDE_NEW + #include <gc/gc.h> #include <gc/gc_cpp.h> @@ -56,6 +58,12 @@ static char * dupStringWithLen(const char * s, size_t size) } +RootValue allocRootValue(Value * v) +{ + return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v); +} + + static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) { checkInterrupt(); @@ -493,52 +501,74 @@ Value & EvalState::getBuiltin(const string & name) LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) { - throw EvalError(format(s) % s2); + throw EvalError(s, s2); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { - throw EvalError(format(s) % s2 % pos); + throw EvalError({ + .hint = hintfmt(s, s2), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) { - throw EvalError(format(s) % s2 % s3); + throw EvalError(s, s2, s3); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { - throw EvalError(format(s) % s2 % s3 % pos); + throw EvalError({ + .hint = hintfmt(s, s2, s3), + .nixCode = NixCode { .errPos = pos } + }); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const Symbol & sym, const Pos & p2)) { - throw EvalError(format(s) % sym % p1 % p2); + // p1 is where the error occurred; p2 is a position mentioned in the message. + throw EvalError({ + .hint = hintfmt(s, sym, p2), + .nixCode = NixCode { .errPos = p1 } + }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { - throw TypeError(format(s) % pos); + throw TypeError({ + .hint = hintfmt(s), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) { - throw TypeError(format(s) % s1); + throw TypeError(s, s1); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) +LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { - throw TypeError(format(s) % fun.showNamePos() % s2 % pos); + throw TypeError({ + .hint = hintfmt(s, fun.showNamePos(), s2), + .nixCode = NixCode { .errPos = pos } + }); } -LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) +LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { - throw AssertionError(format(s) % s1 % pos); + throw AssertionError({ + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } -LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) +LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { - throw UndefinedVarError(format(s) % s1 % pos); + throw UndefinedVarError({ + .hint = hintfmt(s, s1), + .nixCode = NixCode { .errPos = pos } + }); } LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) @@ -606,7 +636,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) return j->value; } if (!env->prevWith) - throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos); + throwUndefinedVarError(var.pos, "undefined variable '%1%'", var.name); for (size_t l = env->prevWith; l; --l, env = env->up) ; } } @@ -804,7 +834,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) Value v; e->eval(*this, env, v); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -918,7 +948,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Symbol nameSym = state.symbols.create(nameVal.string.s); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos); + throwEvalError(i.pos, "dynamic attribute '%1%' already defined at %2%", nameSym, *j->pos); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ @@ -1006,7 +1036,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } else { state.forceAttrs(*vAttrs, pos); if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute '%1%' missing, at %2%", name, pos); + throwEvalError(pos, "attribute '%1%' missing", name); } vAttrs = j->value; pos2 = j->pos; @@ -1132,7 +1162,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } if (fun.type != tLambda) - throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); + throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); @@ -1160,8 +1190,8 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po for (auto & i : lambda.formals->formals) { Bindings::iterator j = arg.attrs->find(i.name); if (j == arg.attrs->end()) { - if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%", - lambda, i.name, pos); + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { attrsUsed++; @@ -1176,7 +1206,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po user. */ for (auto & i : *arg.attrs) if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos); + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); abort(); // can't happen } } @@ -1256,7 +1286,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v) void ExprIf::eval(EvalState & state, Env & env, Value & v) { - (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); + (state.evalBool(env, cond, pos) ? then : else_)->eval(state, env, v); } @@ -1265,7 +1295,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos)) { std::ostringstream out; cond->show(out); - throwAssertionError("assertion '%1%' failed at %2%", out.str(), pos); + throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str()); } body->eval(state, env, v); } @@ -1417,14 +1447,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); + throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == tFloat) { if (vTmp.type == tInt) { nf += vTmp.integer; } else if (vTmp.type == tFloat) { nf += vTmp.fpoint; } else - throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); + throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else s << state.coerceToString(pos, vTmp, context, false, firstType == tString); } @@ -1435,7 +1465,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) mkFloat(v, nf); else if (firstType == tPath) { if (!context.empty()) - throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); + throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); mkPath(v, path.c_str()); } else @@ -1484,7 +1514,7 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type != tInt) - throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1495,16 +1525,16 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) if (v.type == tInt) return v.integer; else if (v.type != tFloat) - throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } bool EvalState::forceBool(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tBool) - throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1517,9 +1547,9 @@ bool EvalState::isFunctor(Value & fun) void EvalState::forceFunction(Value & v, const Pos & pos) { - forceValue(v); + forceValue(v, pos); if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) - throwTypeError("value is %1% while a function was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1528,7 +1558,7 @@ string EvalState::forceString(Value & v, const Pos & pos) forceValue(v, pos); if (v.type != tString) { if (pos) - throwTypeError("value is %1% while a string was expected, at %2%", v, pos); + throwTypeError(pos, "value is %1% while a string was expected", v); else throwTypeError("value is %1% while a string was expected", v); } @@ -1557,8 +1587,8 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) string s = forceString(v, pos); if (v.string.context) { if (pos) - throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%", - v.string.s, v.string.context[0], pos); + throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string.s, v.string.context[0]); else throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]); @@ -1594,7 +1624,7 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v, string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { - forceValue(v); + forceValue(v, pos); string s; @@ -1614,7 +1644,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return *maybeString; } auto i = v.attrs->find(sOutPath); - if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); + if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string"); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } @@ -1645,7 +1675,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } } - throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); + throwTypeError(pos, "cannot coerce %1% to a string", v); } @@ -1661,7 +1691,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) else { auto p = settings.readOnlyMode ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); + : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); @@ -1676,7 +1706,7 @@ Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) { string path = coerceToString(pos, v, context, false, false); if (path == "" || path[0] != '/') - throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos); + throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); return path; } @@ -1883,8 +1913,10 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { - throw TypeError(format("cannot coerce %1% to a string, at %2%") % - showType() % pos); + throw TypeError({ + .hint = hintfmt("cannot coerce %1% to a string", showType()), + .nixCode = NixCode { .errPos = pos } + }); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index eac53b894..1485dc7fe 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -272,6 +272,7 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); + Value * allocAttr(Value & vAttrs, const std::string & name); Bindings * allocBindings(size_t capacity); @@ -367,7 +368,7 @@ struct EvalSettings : Config "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/function-trace.cc b/src/libexpr/function-trace.cc index af1486f78..c6057b384 100644 --- a/src/libexpr/function-trace.cc +++ b/src/libexpr/function-trace.cc @@ -1,4 +1,5 @@ #include "function-trace.hh" +#include "logging.hh" namespace nix { diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 1fdce1983..76e1a26bf 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -4,7 +4,6 @@ #include <nlohmann/json.hpp> using json = nlohmann::json; -using std::unique_ptr; namespace nix { @@ -13,69 +12,69 @@ namespace nix { class JSONSax : nlohmann::json_sax<json> { class JSONState { protected: - unique_ptr<JSONState> parent; - Value * v; + std::unique_ptr<JSONState> parent; + RootValue v; public: - virtual unique_ptr<JSONState> resolve(EvalState &) + virtual std::unique_ptr<JSONState> resolve(EvalState &) { throw std::logic_error("tried to close toplevel json parser state"); - }; - explicit JSONState(unique_ptr<JSONState>&& p) : parent(std::move(p)), v(nullptr) {}; - explicit JSONState(Value* v) : v(v) {}; - JSONState(JSONState& p) = delete; - Value& value(EvalState & state) + } + explicit JSONState(std::unique_ptr<JSONState> && p) : parent(std::move(p)) {} + explicit JSONState(Value * v) : v(allocRootValue(v)) {} + JSONState(JSONState & p) = delete; + Value & value(EvalState & state) { - if (v == nullptr) - v = state.allocValue(); - return *v; - }; - virtual ~JSONState() {}; - virtual void add() {}; + if (!v) + v = allocRootValue(state.allocValue()); + return **v; + } + virtual ~JSONState() {} + virtual void add() {} }; class JSONObjectState : public JSONState { using JSONState::JSONState; - ValueMap attrs = ValueMap(); - virtual unique_ptr<JSONState> resolve(EvalState & state) override + ValueMap attrs; + std::unique_ptr<JSONState> resolve(EvalState & state) override { - Value& v = parent->value(state); + Value & v = parent->value(state); state.mkAttrs(v, attrs.size()); for (auto & i : attrs) v.attrs->push_back(Attr(i.first, i.second)); return std::move(parent); } - virtual void add() override { v = nullptr; }; + void add() override { v = nullptr; } public: - void key(string_t& name, EvalState & state) + void key(string_t & name, EvalState & state) { - attrs[state.symbols.create(name)] = &value(state); + attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; class JSONListState : public JSONState { - ValueVector values = ValueVector(); - virtual unique_ptr<JSONState> resolve(EvalState & state) override + ValueVector values; + std::unique_ptr<JSONState> resolve(EvalState & state) override { - Value& v = parent->value(state); + Value & v = parent->value(state); state.mkList(v, values.size()); for (size_t n = 0; n < values.size(); ++n) { v.listElems()[n] = values[n]; } return std::move(parent); } - virtual void add() override { - values.push_back(v); + void add() override { + values.push_back(*v); v = nullptr; - }; + } public: - JSONListState(unique_ptr<JSONState>&& p, std::size_t reserve) : JSONState(std::move(p)) + JSONListState(std::unique_ptr<JSONState> && p, std::size_t reserve) : JSONState(std::move(p)) { values.reserve(reserve); } }; EvalState & state; - unique_ptr<JSONState> rs; + std::unique_ptr<JSONState> rs; template<typename T, typename... Args> inline bool handle_value(T f, Args... args) { @@ -107,12 +106,12 @@ public: return handle_value(mkInt, val); } - bool number_float(number_float_t val, const string_t& s) + bool number_float(number_float_t val, const string_t & s) { return handle_value(mkFloat, val); } - bool string(string_t& val) + bool string(string_t & val) { return handle_value<void(Value&, const char*)>(mkString, val.c_str()); } @@ -123,7 +122,7 @@ public: return true; } - bool key(string_t& name) + bool key(string_t & name) { dynamic_cast<JSONObjectState*>(rs.get())->key(name, state); return true; diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index c34e5c383..f6e83926b 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -127,14 +127,14 @@ or { return OR_KW; } try { yylval->n = boost::lexical_cast<int64_t>(yytext); } catch (const boost::bad_lexical_cast &) { - throw ParseError(format("invalid integer '%1%'") % yytext); + throw ParseError("invalid integer '%1%'", yytext); } return INT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError(format("invalid float '%1%'") % yytext); + throw ParseError("invalid float '%1%'", yytext); return FLOAT; } @@ -219,4 +219,3 @@ or { return OR_KW; } } %% - diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..9ed39e745 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,7 +6,9 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_LIBS = libutil libstore libnixrust +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr + +libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 63cbef1dd..b4b65883d 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -267,8 +267,11 @@ void ExprVar::bindVars(const StaticEnv & env) /* Otherwise, the variable must be obtained from the nearest enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ - if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); - + if (withLevel == -1) + throw UndefinedVarError({ + .hint = hintfmt("undefined variable '%1%'", name), + .nixCode = NixCode { .errPos = pos } + }); fromWith = true; this->level = withLevel; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index f7e9105a4..ec6fd3190 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -2,6 +2,7 @@ #include "value.hh" #include "symbol-table.hh" +#include "error.hh" #include <map> @@ -209,9 +210,10 @@ struct ExprList : Expr struct Formal { + Pos pos; Symbol name; Expr * def; - Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; + Formal(const Pos & pos, const Symbol & name, Expr * def) : pos(pos), name(name), def(def) { }; }; struct Formals @@ -234,8 +236,10 @@ struct ExprLambda : Expr : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError(format("duplicate formal function argument '%1%' at %2%") - % arg % pos); + throw ParseError({ + .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .nixCode = NixCode { .errPos = pos } + }); }; void setName(Symbol & name); string showNamePos() const; @@ -261,8 +265,9 @@ struct ExprWith : Expr struct ExprIf : Expr { + Pos pos; Expr * cond, * then, * else_; - ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; + ExprIf(const Pos & pos, Expr * cond, Expr * then, Expr * else_) : pos(pos), cond(cond), then(then), else_(else_) { }; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..a639be64e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,7 +31,7 @@ namespace nix { Expr * result; Path basePath; Symbol path; - string error; + ErrorInfo error; Symbol sLetBody; ParseData(EvalState & state) : state(state) @@ -64,15 +64,20 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { - throw ParseError(format("attribute '%1%' at %2% already defined at %3%") - % showAttrPath(attrPath) % pos % prevPos); + throw ParseError({ + .hint = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { - throw ParseError(format("attribute '%1%' at %2% already defined at %3%") - % attr % pos % prevPos); + throw ParseError({ + .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), + .nixCode = NixCode { .errPos = pos }, + }); } @@ -140,8 +145,11 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) - throw ParseError(format("duplicate formal function argument '%1%' at %2%") - % formal.name % pos); + throw ParseError({ + .hint = hintfmt("duplicate formal function argument '%1%'", + formal.name), + .nixCode = NixCode { .errPos = pos }, + }); formals->formals.push_front(formal); } @@ -249,8 +257,10 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { - data->error = (format("%1%, at %2%") - % error % makeCurPos(*loc, data)).str(); + data->error = { + .hint = hintfmt(error), + .nixCode = NixCode { .errPos = makeCurPos(*loc, data) } + }; } @@ -327,15 +337,17 @@ expr_function { $$ = new ExprWith(CUR_POS, $2, $4); } | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) - throw ParseError(format("dynamic attributes not allowed in let at %1%") - % CUR_POS); + throw ParseError({ + .hint = hintfmt("dynamic attributes not allowed in let"), + .nixCode = NixCode { .errPos = CUR_POS }, + }); $$ = new ExprLet($2, $4); } | expr_if ; expr_if - : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } + : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } | expr_op ; @@ -405,7 +417,10 @@ expr_simple | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); if (noURLLiterals) - throw ParseError("URL literals are disabled, at %s", CUR_POS); + throw ParseError({ + .hint = hintfmt("URL literals are disabled"), + .nixCode = NixCode { .errPos = CUR_POS } + }); $$ = new ExprString(data->symbols.create($1)); } | '(' expr ')' { $$ = $2; } @@ -475,8 +490,10 @@ attrs $$->push_back(AttrName(str->s)); delete str; } else - throw ParseError(format("dynamic attributes not allowed in inherit at %1%") - % makeCurPos(@2, data)); + throw ParseError({ + .hint = hintfmt("dynamic attributes not allowed in inherit"), + .nixCode = NixCode { .errPos = makeCurPos(@2, data) }, + }); } | { $$ = new AttrPath; } ; @@ -531,8 +548,8 @@ formals ; formal - : ID { $$ = new Formal(data->symbols.create($1), 0); } - | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } + : ID { $$ = new Formal(CUR_POS, data->symbols.create($1), 0); } + | ID '?' expr { $$ = new Formal(CUR_POS, data->symbols.create($1), $3); } ; %% @@ -544,7 +561,8 @@ formal #include <unistd.h> #include "eval.hh" -#include "download.hh" +#include "filetransfer.hh" +#include "fetchers.hh" #include "store-api.hh" @@ -670,11 +688,13 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } - format f = format( - "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)" - + string(pos ? ", at %2%" : "")); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - throw ThrownError(f % path % pos); + throw ThrownError({ + .hint = hintfmt(evalSettings.pureEval + ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" + : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", + path), + .nixCode = NixCode { .errPos = pos } + }); } @@ -687,11 +707,13 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - CachedDownloadRequest request(elem.second); - request.unpack = true; - res = { true, getDownloader()->downloadCached(store, request).path }; - } catch (DownloadError & e) { - printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); + res = { true, store->toRealPath(fetchers::downloadTarball( + store, resolveUri(elem.second), "source", false).storePath) }; + } catch (FileTransferError & e) { + logWarning({ + .name = "Entry download", + .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + }); res = { false, "" }; } } else { @@ -699,7 +721,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (pathExists(path)) res = { true, path }; else { - printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second); + logWarning({ + .name = "Entry not found", + .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + }); res = { false, "" }; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c9a16784e..925cccbfe 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,6 +1,5 @@ #include "archive.hh" #include "derivations.hh" -#include "download.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -56,7 +55,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!decoded.second.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}}); + drvs.push_back(StorePathWithOutputs{ctx, {decoded.second}}); /* Add the output of this derivation to the allowed paths. */ @@ -94,8 +93,10 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); @@ -122,16 +123,16 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } w.attrs->sort(); - static Value * fun = nullptr; + static RootValue fun; if (!fun) { - fun = state.allocValue(); + fun = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), *fun); + , "/"), **fun); } - state.forceFunction(*fun, pos); - mkApp(v, *fun, w); + state.forceFunction(**fun, pos); + mkApp(v, **fun, w); state.forceAttrs(v, pos); } else { state.forceAttrs(*args[0]); @@ -171,8 +172,12 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt( + "cannot import '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } path = state.checkSourcePath(path); @@ -181,17 +186,17 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - throw EvalError(format("could not open '%1%': %2%") % path % dlerror()); + throw EvalError("could not open '%1%': %2%", path, dlerror()); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message); + throw EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message); else - throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected") - % sym % path); + throw EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", + sym, path); } (func)(state, v); @@ -207,7 +212,10 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) { - throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("at least one argument to 'exec' required"), + .nixCode = NixCode { .errPos = pos } + }); } PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); @@ -218,8 +226,11 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%") - % program % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + program, e.path), + .nixCode = NixCode { .errPos = pos } + }); } auto output = runProgram(program, true, commandArgs); @@ -227,13 +238,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) try { parsed = state.parseExprFromString(output, pos.file); } catch (Error & e) { - e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos); + e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos)); throw; } try { state.eval(parsed, v); } catch (Error & e) { - e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos); + e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos)); throw; } } @@ -242,7 +253,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); string t; switch (args[0]->type) { case tInt: t = "int"; break; @@ -270,7 +281,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is the null value. */ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tNull); } @@ -278,7 +289,7 @@ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Valu /* Determine whether the argument is a function. */ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); bool res; switch (args[0]->type) { case tLambda: @@ -297,21 +308,21 @@ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, /* Determine whether the argument is an integer. */ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tInt); } /* Determine whether the argument is a float. */ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tFloat); } /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tString); } @@ -319,14 +330,14 @@ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a Boolean. */ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tBool); } /* Determine whether the argument is a path. */ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tPath); } @@ -339,7 +350,7 @@ struct CompareValues if (v1->type == tInt && v2->type == tFloat) return v1->integer < v2->fpoint; if (v1->type != v2->type) - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); switch (v1->type) { case tInt: return v1->integer < v2->integer; @@ -350,7 +361,7 @@ struct CompareValues case tPath: return strcmp(v1->path, v2->path) < 0; default: - throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } } }; @@ -371,7 +382,10 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator startSet = args[0]->attrs->find(state.symbols.create("startSet")); if (startSet == args[0]->attrs->end()) - throw EvalError(format("attribute 'startSet' required, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("attribute 'startSet' required"), + .nixCode = NixCode { .errPos = pos } + }); state.forceList(*startSet->value, pos); ValueList workSet; @@ -382,8 +396,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator op = args[0]->attrs->find(state.symbols.create("operator")); if (op == args[0]->attrs->end()) - throw EvalError(format("attribute 'operator' required, at %1%") % pos); - state.forceValue(*op->value); + throw EvalError({ + .hint = hintfmt("attribute 'operator' required"), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of `workSet', adding the result to `workSet', continuing until @@ -401,8 +418,11 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator key = e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) - throw EvalError(format("attribute 'key' required, at %1%") % pos); - state.forceValue(*key->value); + throw EvalError({ + .hint = hintfmt("attribute 'key' required"), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*key->value, pos); if (!doneKeys.insert(key->value).second) continue; res.push_back(e); @@ -414,7 +434,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar /* Add the values returned by the operator to the work set. */ for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n]); + state.forceValue(*call.listElems()[n], pos); workSet.push_back(call.listElems()[n]); } } @@ -431,7 +451,7 @@ static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; string s = state.coerceToString(pos, *args[0], context); - throw Abort(format("evaluation aborted with the following error message: '%1%'") % s); + throw Abort("evaluation aborted with the following error message: '%1%'", s); } @@ -446,7 +466,7 @@ static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { try { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { PathSet context; @@ -462,7 +482,7 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val { state.mkAttrs(v, 2); try { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); v.attrs->push_back(Attr(state.sValue, args[0])); mkBool(*state.allocAttr(v, state.symbols.create("success")), true); } catch (AssertionError & e) { @@ -484,8 +504,8 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -495,7 +515,7 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValueDeep(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -504,12 +524,12 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val return the second expression. Useful for debugging. */ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type == tString) - printError(format("trace: %1%") % args[0]->string.s); + printError("trace: %1%", args[0]->string.s); else - printError(format("trace: %1%") % *args[0]); - state.forceValue(*args[1]); + printError("trace: %1%", *args[0]); + state.forceValue(*args[1], pos); v = *args[1]; } @@ -533,13 +553,16 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Figure out the name first (for stack backtraces). */ Bindings::iterator attr = args[0]->attrs->find(state.sName); if (attr == args[0]->attrs->end()) - throw EvalError(format("required attribute 'name' missing, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("required attribute 'name' missing"), + .nixCode = NixCode { .errPos = pos } + }); string drvName; Pos & posDrvName(*attr->pos); try { drvName = state.forceStringNoCtx(*attr->value, pos); } catch (Error & e) { - e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName); + e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName)); throw; } @@ -563,7 +586,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional<std::string> outputHash; std::string outputHashAlgo; - bool outputHashRecursive = false; + auto ingestionMethod = FileIngestionMethod::Flat; StringSet outputs; outputs.insert("out"); @@ -574,33 +597,46 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string & s) { - if (s == "recursive") outputHashRecursive = true; - else if (s == "flat") outputHashRecursive = false; - else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); + if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; + else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; + else + throw EvalError({ + .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .nixCode = NixCode { .errPos = posDrvName } + }); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName); + throw EvalError({ + .hint = hintfmt("duplicate derivation output '%1%'", j), + .nixCode = NixCode { .errPos = posDrvName } + }); /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName); + throw EvalError({ + .hint = hintfmt("invalid derivation output name 'drv'" ), + .nixCode = NixCode { .errPos = posDrvName } + }); outputs.insert(j); } if (outputs.empty()) - throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + throw EvalError({ + .hint = hintfmt("derivation cannot have an empty set of outputs"), + .nixCode = NixCode { .errPos = posDrvName } + }); }; try { if (ignoreNulls) { - state.forceValue(*i->value); + state.forceValue(*i->value, pos); if (i->value->type == tNull) continue; } @@ -687,9 +723,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * StorePathSet refs; state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); for (auto & j : refs) { - drv.inputSrcs.insert(j.clone()); + drv.inputSrcs.insert(j); if (j.isDerivation()) - drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j); + drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); } } @@ -706,27 +742,44 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") - throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName); + throw EvalError({ + .hint = hintfmt("required attribute 'builder' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); + if (drv.platform == "") - throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName); + throw EvalError({ + .hint = hintfmt("required attribute 'system' missing"), + .nixCode = NixCode { .errPos = posDrvName } + }); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) - throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName); + throw EvalError({ + .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .nixCode = NixCode { .errPos = posDrvName } + }); if (outputHash) { /* Handle fixed-output derivations. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); + throw Error({ + .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .nixCode = NixCode { .errPos = posDrvName } + }); HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); - Hash h(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + Hash h = newHashAllowEmpty(*outputHash, ht); + + auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath), - (outputHashRecursive ? "r:" : "") + printHashType(h.type), - h.to_string(Base16, false))); + drv.outputs.insert_or_assign("out", DerivationOutput { + std::move(outPath), + (ingestionMethod == FileIngestionMethod::Recursive ? "r:" : "") + + printHashType(h.type), + h.to_string(Base16, false), + }); } else { @@ -739,7 +792,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { if (!jsonObject) drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput(StorePath::dummy.clone(), "", "")); + DerivationOutput { StorePath::dummy, "", "" }); } // Regular, non-CA derivation should always return a single hash and not @@ -750,7 +803,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto outPath = state.store->makeOutputPath(i, h, drvName); if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign(i, - DerivationOutput(std::move(outPath), "", "")); + DerivationOutput { std::move(outPath), "", "" }); } } @@ -763,7 +816,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't read them later. */ - drvHashes.insert_or_assign(drvPath.clone(), + drvHashes.insert_or_assign(drvPath, hashDerivationModulo(*state.store, Derivation(drv), false)); state.mkAttrs(v, 1 + drv.outputs.size()); @@ -820,7 +873,10 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V e.g. nix-push does the right thing. */ if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) - throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos); + throw EvalError({ + .hint = hintfmt("path '%1%' is not in the Nix store", path), + .nixCode = NixCode { .errPos = pos } + }); Path path2 = state.store->toStorePath(path); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(path2)); @@ -836,9 +892,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format( - "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt( + "cannot check the existence of '%1%', since path '%2%' is not valid", + path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } try { @@ -881,12 +940,14 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); if (s.find((char) 0) != string::npos) - throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path); + throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); mkString(v, s.c_str()); } @@ -910,7 +971,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) - throw EvalError(format("attribute 'path' missing, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("attribute 'path' missing"), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -918,8 +982,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va try { state.realiseContext(context); } catch (InvalidPathError & e) { - throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } searchPath.emplace_back(prefix, path); @@ -936,7 +1002,10 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + throw Error({ + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded Path p = state.coerceToPath(pos, *args[1], context); @@ -952,8 +1021,10 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val try { state.realiseContext(ctx); } catch (InvalidPathError & e) { - throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") - % path % e.path % pos); + throw EvalError({ + .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .nixCode = NixCode { .errPos = pos } + }); } DirEntries entries = readDirectory(state.checkSourcePath(path)); @@ -1023,7 +1094,13 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); + throw EvalError( { + .hint = hintfmt( + "in 'toFile': the file named '%1%' must not contain a reference " + "to a derivation but contains (%2%)", + name, path), + .nixCode = NixCode { .errPos = pos } + }); refs.insert(state.store->parseStorePath(path)); } @@ -1040,7 +1117,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1071,12 +1148,12 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first - : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair)); + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else @@ -1091,13 +1168,21 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args PathSet context; Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + throw EvalError({ + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = pos } + }); - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) - throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + throw TypeError({ + .hint = hintfmt( + "first argument in call to 'filterSource' is not a function but %1%", + showType(*args[0])), + .nixCode = NixCode { .errPos = pos } + }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1106,7 +1191,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Path path; string name; Value * filterFun = nullptr; - auto recursive = true; + auto method = FileIngestionMethod::Recursive; Hash expectedHash; for (auto & attr : *args[0]->attrs) { @@ -1115,25 +1200,34 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value PathSet context; path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + throw EvalError({ + .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .nixCode = NixCode { .errPos = *attr.pos } + }); } else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { - state.forceValue(*attr.value); + state.forceValue(*attr.value, pos); filterFun = attr.value; } else if (n == "recursive") - recursive = state.forceBool(*attr.value, *attr.pos); + method = FileIngestionMethod { state.forceBool(*attr.value, *attr.pos) }; else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else - throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + throw EvalError({ + .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (path.empty()) - throw EvalError(format("'path' required, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("'path' required"), + .nixCode = NixCode { .errPos = pos } + }); if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v); } @@ -1187,10 +1281,13 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); + throw EvalError({ + .hint = hintfmt("attribute '%1%' missing", attr), + .nixCode = NixCode { .errPos = pos } + }); // !!! add to stack trace? if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; - state.forceValue(*i->value); + state.forceValue(*i->value, pos); v = *i->value; } @@ -1220,7 +1317,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val /* Determine whether the argument is a set. */ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->type == tAttrs); } @@ -1267,15 +1364,20 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Bindings::iterator j = v2.attrs->find(state.sName); if (j == v2.attrs->end()) - throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos); + throw TypeError({ + .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) - throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); - + throw TypeError({ + .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), + .nixCode = NixCode { .errPos = pos } + }); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } } @@ -1346,9 +1448,12 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va */ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type != tLambda) - throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); + throw TypeError({ + .hint = hintfmt("'functionArgs' requires a function"), + .nixCode = NixCode { .errPos = pos } + }); if (!args[0]->lambda.fun->matchAttrs) { state.mkAttrs(v, 0); @@ -1356,9 +1461,12 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args } state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) + for (auto & i : args[0]->lambda.fun->formals->formals) { // !!! should optimise booleans (allocate only once) - mkBool(*state.allocAttr(v, i.name), i.def); + Value * value = state.allocValue(); + v.attrs->push_back(Attr(i.name, value, &i.pos)); + mkBool(*value, i.def); + } v.attrs->sort(); } @@ -1389,7 +1497,7 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va /* Determine whether the argument is a list. */ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); mkBool(v, args[0]->isList()); } @@ -1398,8 +1506,11 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu { state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) - throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); - state.forceValue(*list.listElems()[n]); + throw Error({ + .hint = hintfmt("list index %1% is out of bounds", n), + .nixCode = NixCode { .errPos = pos } + }); + state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -1425,7 +1536,11 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value { state.forceList(*args[0], pos); if (args[0]->listSize() == 0) - throw Error(format("'tail' called on an empty list, at %1%") % pos); + throw Error({ + .hint = hintfmt("'tail' called on an empty list"), + .nixCode = NixCode { .errPos = pos } + }); + state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) v.listElems()[n] = args[0]->listElems()[n + 1]; @@ -1522,9 +1637,9 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); } - state.forceValue(v); + state.forceValue(v, pos); } else { - state.forceValue(*args[1]); + state.forceValue(*args[1], pos); v = *args[1]; } } @@ -1566,7 +1681,10 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val auto len = state.forceInt(*args[1], pos); if (len < 0) - throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos); + throw EvalError({ + .hint = hintfmt("cannot create list of size %1%", len), + .nixCode = NixCode { .errPos = pos } + }); state.mkList(v, len); @@ -1589,7 +1707,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto len = args[1]->listSize(); state.mkList(v, len); for (unsigned int n = 0; n < len; ++n) { - state.forceValue(*args[1]->listElems()[n]); + state.forceValue(*args[1]->listElems()[n], pos); v.listElems()[n] = args[1]->listElems()[n]; } @@ -1624,7 +1742,7 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V for (unsigned int n = 0; n < len; ++n) { auto vElem = args[1]->listElems()[n]; - state.forceValue(*vElem); + state.forceValue(*vElem, pos); Value res; state.callFunction(*args[0], *vElem, res, pos); if (state.forceBool(res, pos)) @@ -1724,7 +1842,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[1], pos); NixFloat f2 = state.forceFloat(*args[1], pos); - if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); + if (f2 == 0) + throw EvalError({ + .hint = hintfmt("division by zero"), + .nixCode = NixCode { .errPos = pos } + }); if (args[0]->type == tFloat || args[1]->type == tFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); @@ -1733,7 +1855,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixInt i2 = state.forceInt(*args[1], pos); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) - throw EvalError(format("overflow in integer division, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("overflow in integer division"), + .nixCode = NixCode { .errPos = pos } + }); + mkInt(v, i1 / i2); } } @@ -1755,8 +1881,8 @@ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Valu static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.forceValue(*args[0]); - state.forceValue(*args[1]); + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); CompareValues comp; mkBool(v, comp(args[0], args[1])); } @@ -1789,7 +1915,11 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V PathSet context; string s = state.coerceToString(pos, *args[2], context); - if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos); + if (start < 0) + throw EvalError({ + .hint = hintfmt("negative start position in 'substring'"), + .nixCode = NixCode { .errPos = pos } + }); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } @@ -1809,7 +1939,10 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, string type = state.forceStringNoCtx(*args[0], pos); HashType ht = parseHashType(type); if (ht == htUnknown) - throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + throw Error({ + .hint = hintfmt("unknown hash type '%1%'", type), + .nixCode = NixCode { .errPos = pos } + }); PathSet context; // discarded string s = state.forceString(*args[1], context, pos); @@ -1851,10 +1984,16 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError({ + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } else { - throw EvalError("invalid regular expression '%s', at %s", re, pos); + throw EvalError({ + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } } } @@ -1918,10 +2057,16 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value } catch (std::regex_error &e) { if (e.code() == std::regex_constants::error_space) { - // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError({ + .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } else { - throw EvalError("invalid regular expression '%s', at %s", re, pos); + throw EvalError({ + .hint = hintfmt("invalid regular expression '%s'", re), + .nixCode = NixCode { .errPos = pos } + }); } } } @@ -1952,7 +2097,10 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[0], pos); state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) - throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .nixCode = NixCode { .errPos = pos } + }); vector<string> from; from.reserve(args[0]->listSize()); @@ -2048,68 +2196,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args /************************************************************* - * Networking - *************************************************************/ - - -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, const std::string & defaultName) -{ - CachedDownloadRequest request(""); - request.unpack = unpack; - request.name = defaultName; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") - request.name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); - } - - if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); - - } else - request.uri = state.forceStringNoCtx(*args[0], pos); - - state.checkURI(request.uri); - - if (evalSettings.pureEval && !request.expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - - auto res = getDownloader()->downloadCached(state.store, request); - - if (state.allowedPaths) - state.allowedPaths->insert(res.path); - - mkString(v, res.storePath, PathSet({res.storePath})); -} - - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchurl", false, ""); -} - - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true, "source"); -} - - -/************************************************************* * Primop registration *************************************************************/ @@ -2291,10 +2377,6 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); addPrimOp("placeholder", 1, prim_placeholder); - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index c790b30f6..05d0792ef 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -20,6 +20,7 @@ struct RegisterPrimOp them. */ /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); + /* Execute a program and parse its output */ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 94fa0158c..efa2e9576 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -146,7 +146,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg auto sAllOutputs = state.symbols.create("allOutputs"); for (auto & i : *args[1]->attrs) { if (!state.store->isStorePath(i.name)) - throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + throw EvalError({ + .hint = hintfmt("Context key '%s' is not a store path", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); if (!settings.readOnlyMode) state.store->ensurePath(state.store->parseStorePath(i.name)); state.forceAttrs(*i.value, *i.pos); @@ -160,7 +163,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, *iter->pos)) { if (!isDerivation(i.name)) { - throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + throw EvalError({ + .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); } context.insert("=" + string(i.name)); } @@ -170,7 +176,10 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (iter != i.value->attrs->end()) { state.forceList(*iter->value, *iter->pos); if (iter->value->listSize() && !isDerivation(i.name)) { - throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + throw EvalError({ + .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .nixCode = NixCode { .errPos = *i.pos } + }); } for (unsigned int n = 0; n < iter->value->listSize(); ++n) { auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..dd7229a3d 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -1,203 +1,19 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" #include "hash.hh" -#include "tarfile.hh" - -#include <sys/time.h> - -#include <regex> - -#include <nlohmann/json.hpp> - -using namespace std::string_literals; +#include "fetchers.hh" +#include "url.hh" namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; -}; - -std::regex revRegex("^[0-9a-fA-F]{40}$"); - -GitInfo exportGit(ref<Store> store, const std::string & uri, - std::optional<std::string> ref, std::string rev, - const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { - - bool clean = true; - - try { - runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - clean = false; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - auto files = tokenizeString<std::set<std::string>>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return gitInfo; - } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; - } - - if (!ref) ref = "HEAD"s; - - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); - - deletePath(getCacheDir() + "/nix/git"); - - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); - - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } - - Path localRefFile; - if (ref->compare(0, 5, "refs/") == 0) - localRefFile = cacheDir + "/" + *ref; - else - localRefFile = cacheDir + "/refs/heads/" + *ref; - - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev != "") { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); - - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); - } - - // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); - Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == gitInfo.rev); - - gitInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) { - gitInfo.revCount = json["revCount"]; - return gitInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - unpackTarfile(*source, tmpDir); - - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); - - nlohmann::json json; - json["storePath"] = gitInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["rev"] = gitInfo.rev; - json["revCount"] = gitInfo.revCount; - - writeFile(storeLink, json.dump()); - - return gitInfo; -} - static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; std::optional<std::string> ref; - std::string rev; + std::optional<Hash> rev; std::string name = "source"; + bool fetchSubmodules = false; PathSet context; state.forceValue(*args[0]); @@ -213,15 +29,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "submodules") + fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (url.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.coerceToString(pos, *args[0], context, false, false); @@ -230,17 +54,36 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + if (ref) attrs.insert_or_assign("ref", *ref); + if (rev) attrs.insert_or_assign("rev", rev->gitRev()); + if (fetchSubmodules) attrs.insert_or_assign("submodules", true); + auto input = fetchers::inputFromAttrs(attrs); + + // FIXME: use name? + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); + // Backward compatibility: set 'revCount' to 0 for a dirty tree. + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), + tree.info.revCount.value_or(0)); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index db274fa4f..9bace8f89 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,174 +1,18 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" - -#include <sys/time.h> +#include "fetchers.hh" +#include "url.hh" #include <regex> -#include <nlohmann/json.hpp> - -using namespace std::string_literals; - namespace nix { -struct HgInfo -{ - Path storePath; - std::string branch; - std::string rev; - uint64_t revCount = 0; -}; - -std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); - -HgInfo exportMercurial(ref<Store> store, const std::string & uri, - std::string rev, const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { - - bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked - files. */ - - printTalkative("copying unclean Mercurial working tree '%s'", uri); - - HgInfo hgInfo; - hgInfo.rev = "0000000000000000000000000000000000000000"; - hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); - - auto files = tokenizeString<std::set<std::string>>( - runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return hgInfo; - } - } - - if (rev == "") rev = "default"; - - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); - - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); - - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) - { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(std::regex_match(rev, commitHashRegex) - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); - - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); - } - } - - writeFile(stampFile, ""); - } - - auto tokens = tokenizeString<std::vector<std::string>>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); - assert(tokens.size() == 3); - - HgInfo hgInfo; - hgInfo.rev = tokens[0]; - hgInfo.revCount = std::stoull(tokens[1]); - hgInfo.branch = tokens[2]; - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == hgInfo.rev); - - hgInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) { - printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); - return hgInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); - - deletePath(tmpDir + "/.hg_archival.txt"); - - hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - nlohmann::json json; - json["storePath"] = hgInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["branch"] = hgInfo.branch; - json["rev"] = hgInfo.rev; - json["revCount"] = hgInfo.revCount; - - writeFile(storeLink, json.dump()); - - return hgInfo; -} - static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::string rev; + std::optional<Hash> rev; + std::optional<std::string> ref; std::string name = "source"; PathSet context; @@ -182,16 +26,29 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar string n(attr.name); if (n == "url") url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") { + // Ugly: unlike fetchGit, here the "rev" attribute can + // be both a revision or a branch/tag name. + auto value = state.forceStringNoCtx(*attr.value, *attr.pos); + if (std::regex_match(value, revRegex)) + rev = Hash(value, htSHA1); + else + ref = value; + } else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else - throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos); + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .nixCode = NixCode { .errPos = *attr.pos } + }); } if (url.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); } else url = state.coerceToString(pos, *args[0], context, false, false); @@ -200,18 +57,35 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // whitelist. Ah well. state.checkURI(url); - auto hgInfo = exportMercurial(state.store, url, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "hg"); + attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + if (ref) attrs.insert_or_assign("ref", *ref); + if (rev) attrs.insert_or_assign("rev", rev->gitRev()); + auto input = fetchers::inputFromAttrs(attrs); + + // FIXME: use name + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); - mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + if (input2->getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc new file mode 100644 index 000000000..9be93710a --- /dev/null +++ b/src/libexpr/primops/fetchTree.cc @@ -0,0 +1,172 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "filetransfer.hh" + +#include <ctime> +#include <iomanip> + +namespace nix { + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr<const fetchers::Input> input, + Value & v) +{ + state.mkAttrs(v, 8); + + auto storePath = state.store->printStorePath(tree.storePath); + + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + + assert(tree.info.narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + tree.info.narHash.to_string(SRI, true)); + + if (input->getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + } + + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + + if (tree.info.lastModified) + mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + + v.attrs->sort(); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + + std::shared_ptr<const fetchers::Input> input; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + fetchers::Attrs attrs; + + for (auto & attr : *args[0]->attrs) { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else if (attr.value->type == tBool) + attrs.emplace(attr.name, attr.value->boolean); + else + throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + attr.name, showType(*attr.value)); + } + + if (!attrs.count("type")) + throw Error({ + .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .nixCode = NixCode { .errPos = pos } + }); + + input = fetchers::inputFromAttrs(attrs); + } else + input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + + if (evalSettings.pureEval && !input->isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + + // FIXME: use fetchOrSubstituteTree + auto [tree, input2] = input->fetchTree(state.store); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + emitTreeAttrs(state, tree, input2, v); +} + +static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); + +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional<std::string> url; + std::optional<Hash> expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError({ + .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), + .nixCode = NixCode { .errPos = *attr.pos } + }); + } + + if (!url) + throw EvalError({ + .hint = hintfmt("'url' argument required"), + .nixCode = NixCode { .errPos = pos } + }); + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + +} diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index a84e569e9..7615d1379 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,7 +1,7 @@ #include "primops.hh" #include "eval-inline.hh" -#include "cpptoml/cpptoml.h" +#include "../../cpptoml/cpptoml.h" namespace nix { @@ -81,7 +81,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va try { visit(v, parser(tomlStream).parse()); } catch (std::runtime_error & e) { - throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + throw EvalError({ + .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .nixCode = NixCode { .errPos = pos } + }); } } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 5fe8570ad..6ec8315ba 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -79,7 +79,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; default: - throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + throw TypeError("cannot convert %1% to JSON", showType(v)); } } @@ -93,7 +93,7 @@ void printValueAsJSON(EvalState & state, bool strict, void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, JSONPlaceholder & out, PathSet & context) const { - throw TypeError(format("cannot convert %1% to JSON") % showType()); + throw TypeError("cannot convert %1% to JSON", showType()); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 689373873..71025824e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -253,12 +253,17 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC -typedef std::vector<Value *, gc_allocator<Value *> > ValueVector; -typedef std::map<Symbol, Value *, std::less<Symbol>, gc_allocator<std::pair<const Symbol, Value *> > > ValueMap; +typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector; +typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap; #else typedef std::vector<Value *> ValueVector; typedef std::map<Symbol, Value *> ValueMap; #endif +/* A value allocated in traceable memory. */ +typedef std::shared_ptr<Value *> RootValue; + +RootValue allocRootValue(Value * v); + } diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc new file mode 100644 index 000000000..feb0a6085 --- /dev/null +++ b/src/libfetchers/attrs.cc @@ -0,0 +1,107 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get<int64_t>()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get<std::string>()); + else if (i.value().is_boolean()) + attrs.emplace(i.key(), i.value().get<bool>()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if<int64_t>(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if<std::string>(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) { + json[attr.first] = v->t; + } else abort(); + } + return json; +} + +std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if<std::string>(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if<int64_t>(&i->second)) + return *v; + throw Error("input attribute '%s' is not an integer", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if<int64_t>(&i->second)) + return *v; + throw Error("input attribute '%s' is not a Boolean", name); +} + +bool getBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetBoolAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::map<std::string, std::string> attrsToQuery(const Attrs & attrs) +{ + std::map<std::string, std::string> query; + for (auto & attr : attrs) { + if (auto v = std::get_if<int64_t>(&attr.second)) { + query.insert_or_assign(attr.first, fmt("%d", *v)); + } else if (auto v = std::get_if<std::string>(&attr.second)) { + query.insert_or_assign(attr.first, *v); + } else if (auto v = std::get_if<Explicit<bool>>(&attr.second)) { + query.insert_or_assign(attr.first, v->t ? "1" : "0"); + } else abort(); + } + return query; +} + +} diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh new file mode 100644 index 000000000..d6e0ae000 --- /dev/null +++ b/src/libfetchers/attrs.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +#include <variant> + +#include <nlohmann/json_fwd.hpp> + +namespace nix::fetchers { + +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template<typename T> +struct Explicit { + T t; +}; + +typedef std::variant<std::string, int64_t, Explicit<bool>> Attr; +typedef std::map<std::string, Attr> Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name); + +bool getBoolAttr(const Attrs & attrs, const std::string & name); + +std::map<std::string, std::string> attrsToQuery(const Attrs & attrs); + +} diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc new file mode 100644 index 000000000..e1c7f3dee --- /dev/null +++ b/src/libfetchers/cache.cc @@ -0,0 +1,121 @@ +#include "cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync<State> _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref<Store> store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional<std::pair<Attrs, StorePath>> lookup( + ref<Store> store, + const Attrs & inAttrs) override + { + if (auto res = lookupExpired(store, inAttrs)) { + if (!res->expired) + return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); + debug("ignoring expired cache entry '%s'", + attrsToJson(inAttrs).dump()); + } + return {}; + } + + std::optional<Result> lookupExpired( + ref<Store> store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return Result { + .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .storePath = std::move(storePath) + }; + } +}; + +ref<Cache> getCache() +{ + static auto cache = std::make_shared<CacheImpl>(); + return ref<Cache>(cache); +} + +} diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh new file mode 100644 index 000000000..d76ab1233 --- /dev/null +++ b/src/libfetchers/cache.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref<Store> store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional<std::pair<Attrs, StorePath>> lookup( + ref<Store> store, + const Attrs & inAttrs) = 0; + + struct Result + { + bool expired = false; + Attrs infoAttrs; + StorePath storePath; + }; + + virtual std::optional<Result> lookupExpired( + ref<Store> store, + const Attrs & inAttrs) = 0; +}; + +ref<Cache> getCache(); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc new file mode 100644 index 000000000..11cac4c55 --- /dev/null +++ b/src/libfetchers/fetchers.cc @@ -0,0 +1,75 @@ +#include "fetchers.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr<Input> inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr<Input> inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("narHash"); + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs2); + if (res) { + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = newHashAllowEmpty(*narHash, htUnknown); + return res; + } + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI, true)); + attrs.emplace("type", type()); + return attrs; +} + +std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(tree.storePath); + + if (!tree.info.narHash) + tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.info.narHash); + + if (narHash && narHash != input->narHash) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); + + return {std::move(tree), input}; +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh new file mode 100644 index 000000000..59a58ae67 --- /dev/null +++ b/src/libfetchers/fetchers.hh @@ -0,0 +1,103 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" +#include "tree-info.hh" +#include "attrs.hh" +#include "url.hh" + +#include <memory> + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + TreeInfo info; +}; + +struct Input : std::enable_shared_from_this<Input> +{ + std::optional<Hash> narHash; // FIXME: implement + + virtual std::string type() const = 0; + + virtual ~Input() { } + + virtual bool operator ==(const Input & other) const { return false; } + + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + virtual bool isDirect() const { return true; } + + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional<std::string> getRef() const { return {}; } + + virtual std::optional<Hash> getRev() const { return {}; } + + virtual ParsedURL toURL() const = 0; + + std::string to_string() const + { + return toURL().to_string(); + } + + Attrs toAttrs() const; + + std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; + +private: + + virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; +}; + +struct InputScheme +{ + virtual ~InputScheme() { } + + virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0; +}; + +std::unique_ptr<Input> inputFromURL(const ParsedURL & url); + +std::unique_ptr<Input> inputFromURL(const std::string & url); + +std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs); + +void registerInputScheme(std::unique_ptr<InputScheme> && fetcher); + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; +}; + +DownloadFileResult downloadFile( + ref<Store> store, + const std::string & url, + const std::string & name, + bool immutable); + +Tree downloadTarball( + ref<Store> store, + const std::string & url, + const std::string & name, + bool immutable); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc new file mode 100644 index 000000000..75ce5ee8b --- /dev/null +++ b/src/libfetchers/git.cc @@ -0,0 +1,441 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include <sys/time.h> + +using namespace std::string_literals; + +namespace nix::fetchers { + +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + +static bool isNotDotGitDirectory(const Path & path) +{ + static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); + + return not std::regex_match(path, gitDirRegex); +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional<std::string> ref; + std::optional<Hash> rev; + bool shallow = false; + bool submodules = false; + + GitInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "git"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const GitInput *>(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional<std::string> getRef() const override { return ref; } + + std::optional<Hash> getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); + if (submodules) + attrs.emplace("submodules", true); + return attrs; + } + + std::pair<bool, std::string> getActualUrl() const + { + // Don't clone file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + auto name = "source"; + + auto input = std::make_shared<GitInput>(*this); + + assert(!rev || rev->type == htSHA1); + + std::string cacheType = "git"; + if (shallow) cacheType += "-shallow"; + if (submodules) cacheType += "-submodules"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair<Tree, std::shared_ptr<const Input>> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + + if (rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" }); + if (submodules) + gitOpts.emplace_back("--recurse-submodules"); + + auto files = tokenizeString<std::set<std::string>>( + runProgram("git", true, gitOpts), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + auto fetchRef = input->ref->compare(0, 5, "refs/") == 0 + ? *input->ref + : "refs/heads/" + *input->ref; + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; + + if (submodules) { + Path tmpGitDir = createTempDir(); + AutoDelete delTmpGitDir(tmpGitDir, true); + + runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + // TODO: repoDir might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch + // everything to ensure we get the rev. + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + + filter = isNotDotGitDirectory; + } else { + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + unpackTarfile(*source, tmpDir); + } + + auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); + + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") + throw Error("unsupported Git input attribute '%s'", name); + + auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (std::regex_search(*ref, badGitRefRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + + input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); }); + +} diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc new file mode 100644 index 000000000..0bee1d6b3 --- /dev/null +++ b/src/libfetchers/github.cc @@ -0,0 +1,195 @@ +#include "filetransfer.hh" +#include "cache.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional<std::string> ref; + std::optional<Hash> rev; + + std::string type() const override { return "github"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const GitHubInput *>(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional<std::string> getRef() const override { return ref; } + + std::optional<Hash> getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto path = owner + "/" + repo; + assert(!(ref && rev)); + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "github", + .path = path, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + auto rev = this->rev; + auto ref = this->ref.value_or("master"); + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + rev = Hash(std::string { json["sha"] }, htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + auto input = std::make_shared<GitHubInput>(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + auto tree = downloadTarball(store, url, "source", true); + + getCache()->add( + store, + immutableAttrs, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, + true); + + return {std::move(tree), input}; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + auto input = std::make_unique<GitHubInput>(); + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitHub input attribute '%s'", name); + + auto input = std::make_unique<GitHubInput>(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); }); + +} diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk new file mode 100644 index 000000000..cfd705e22 --- /dev/null +++ b/src/libfetchers/local.mk @@ -0,0 +1,11 @@ +libraries += libfetchers + +libfetchers_NAME = libnixfetchers + +libfetchers_DIR := $(d) + +libfetchers_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers_CXXFLAGS += -I src/libutil -I src/libstore + +libfetchers_LIBS = libutil libstore diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc new file mode 100644 index 000000000..2e0d4bf4d --- /dev/null +++ b/src/libfetchers/mercurial.cc @@ -0,0 +1,303 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include <sys/time.h> + +using namespace std::string_literals; + +namespace nix::fetchers { + +struct MercurialInput : Input +{ + ParsedURL url; + std::optional<std::string> ref; + std::optional<Hash> rev; + + MercurialInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "hg"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const MercurialInput *>(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional<std::string> getRef() const override { return ref; } + + std::optional<Hash> getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + url2.scheme = "hg+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::pair<bool, std::string> getActualUrl() const + { + bool isLocal = url.scheme == "file"; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + auto name = "source"; + + auto input = std::make_shared<MercurialInput>(*this); + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // FIXME: return lastModified. + + // FIXME: don't clone local repositories. + + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + if (!settings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", actualUrl); + + if (settings.warnDirty) + warn("Mercurial tree '%s' is unclean", actualUrl); + + input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + + auto files = tokenizeString<std::set<std::string>>( + runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input}; + } + } + + if (!input->ref) input->ref = "default"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair<Tree, std::shared_ptr<const Input>> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + assert(input->rev || input->ref); + auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + } + } + + auto tokens = tokenizeString<std::vector<std::string>>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + input->rev = Hash(tokens[0], htSHA1); + auto revCount = std::stoull(tokens[1]); + input->ref = tokens[2]; + + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + auto storePath = store->addToStore(name, tmpDir); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", (int64_t) revCount}, + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct MercurialInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return nullptr; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Mercurial input attribute '%s'", name); + + auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); }); + +} diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc new file mode 100644 index 000000000..ba2cc192e --- /dev/null +++ b/src/libfetchers/path.cc @@ -0,0 +1,148 @@ +#include "fetchers.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +struct PathInput : Input +{ + Path path; + + /* Allow the user to pass in "fake" tree info attributes. This is + useful for making a pinned tree work the same as the repository + from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + std::optional<Hash> rev; + std::optional<uint64_t> revCount; + std::optional<time_t> lastModified; + + std::string type() const override { return "path"; } + + std::optional<Hash> getRev() const override { return rev; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const PathInput *>(&other); + return + other2 + && path == other2->path + && rev == other2->rev + && revCount == other2->revCount + && lastModified == other2->lastModified; + } + + bool isImmutable() const override + { + return (bool) narHash; + } + + ParsedURL toURL() const override + { + auto query = attrsToQuery(toAttrsInternal()); + query.erase("path"); + return ParsedURL { + .scheme = "path", + .path = path, + .query = query, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("path", path); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (revCount) + attrs.emplace("revCount", *revCount); + if (lastModified) + attrs.emplace("lastModified", *lastModified); + return attrs; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + auto input = std::make_shared<PathInput>(*this); + + // FIXME: check whether access to 'path' is allowed. + + auto storePath = store->maybeParseStorePath(path); + + if (storePath) + store->addTempRoot(*storePath); + + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + // FIXME: try to substitute storePath. + storePath = store->addToStore("source", path); + + return + { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = revCount, + .lastModified = lastModified + } + }, + input + }; + } + +}; + +struct PathInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return nullptr; + + auto input = std::make_unique<PathInput>(); + input->path = url.path; + + for (auto & [name, value] : url.query) + if (name == "rev") + input->rev = Hash(value, htSHA1); + else if (name == "revCount") { + uint64_t revCount; + if (!string2Int(value, revCount)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->revCount = revCount; + } + else if (name == "lastModified") { + time_t lastModified; + if (!string2Int(value, lastModified)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->lastModified = lastModified; + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + auto input = std::make_unique<PathInput>(); + input->path = getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + if (name == "rev") + input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); + else if (name == "revCount") + input->revCount = getIntAttr(attrs, "revCount"); + else if (name == "lastModified") + input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "type" || name == "path") + ; + else + throw Error("unsupported path input attribute '%s'", name); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); }); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc new file mode 100644 index 000000000..7966da314 --- /dev/null +++ b/src/libfetchers/tarball.cc @@ -0,0 +1,275 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "filetransfer.hh" +#include "globals.hh" +#include "store-api.hh" +#include "archive.hh" +#include "tarfile.hh" + +namespace nix::fetchers { + +DownloadFileResult downloadFile( + ref<Store> store, + const std::string & url, + const std::string & name, + bool immutable) +{ + // FIXME: check store + + Attrs inAttrs({ + {"type", "file"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + auto useCached = [&]() -> DownloadFileResult + { + return { + .storePath = std::move(cached->storePath), + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + }; + }; + + if (cached && !cached->expired) + return useCached(); + + FileTransferRequest request(url); + if (cached) + request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); + FileTransferResult res; + try { + res = getFileTransfer()->download(request); + } catch (FileTransferError & e) { + if (cached) { + warn("%s; using cached version", e.msg()); + return useCached(); + } else + throw; + } + + // FIXME: write to temporary file. + + Attrs infoAttrs({ + {"etag", res.etag}, + {"url", res.effectiveUri}, + }); + + std::optional<StorePath> storePath; + + if (res.cached) { + assert(cached); + assert(request.expectedETag == res.etag); + storePath = std::move(cached->storePath); + } else { + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); + auto source = StringSource { *sink.s }; + store->addToStore(info, source, NoRepair, NoCheckSigs); + storePath = std::move(info.path); + } + + getCache()->add( + store, + inAttrs, + infoAttrs, + *storePath, + immutable); + + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + + return { + .storePath = std::move(*storePath), + .etag = res.etag, + .effectiveUrl = res.effectiveUri, + }; +} + +Tree downloadTarball( + ref<Store> store, + const std::string & url, + const std::string & name, + bool immutable) +{ + Attrs inAttrs({ + {"type", "tarball"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + if (cached && !cached->expired) + return Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), + .info = TreeInfo { + .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + }, + }; + + auto res = downloadFile(store, url, name, immutable); + + std::optional<StorePath> unpackedStorePath; + time_t lastModified; + + if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { + unpackedStorePath = std::move(cached->storePath); + lastModified = getIntAttr(cached->infoAttrs, "lastModified"); + } else { + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(res.storePath), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + } + + Attrs infoAttrs({ + {"lastModified", lastModified}, + {"etag", res.etag}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + *unpackedStorePath, + immutable); + + return Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), + .info = TreeInfo { + .lastModified = lastModified, + }, + }; +} + +struct TarballInput : Input +{ + ParsedURL url; + std::optional<Hash> hash; + + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const TarballInput *>(&other); + return + other2 + && to_string() == other2->to_string() + && hash == other2->hash; + } + + bool isImmutable() const override + { + return hash || narHash; + } + + ParsedURL toURL() const override + { + auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (narHash) + url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (hash) + attrs.emplace("hash", hash->to_string(SRI, true)); + return attrs; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + auto tree = downloadTarball(store, url.to_string(), "source", false); + + auto input = std::make_shared<TarballInput>(*this); + input->narHash = store->queryPathInfo(tree.storePath)->narHash; + + return {std::move(tree), input}; + } +}; + +struct TarballInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + + if (!hasSuffix(url.path, ".zip") + && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tar.gz") + && !hasSuffix(url.path, ".tar.xz") + && !hasSuffix(url.path, ".tar.bz2")) + return nullptr; + + auto input = std::make_unique<TarballInput>(url); + + auto hash = input->url.query.find("hash"); + if (hash != input->url.query.end()) { + // FIXME: require SRI hash. + input->hash = Hash(hash->second); + input->url.query.erase(hash); + } + + auto narHash = input->url.query.find("narHash"); + if (narHash != input->url.query.end()) { + // FIXME: require SRI hash. + input->narHash = Hash(narHash->second); + input->url.query.erase(narHash); + } + + return input; + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "hash") + throw Error("unsupported tarball input attribute '%s'", name); + + auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + input->hash = newHashAllowEmpty(*hash, htUnknown); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); + +} diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc new file mode 100644 index 000000000..b2d8cfc8d --- /dev/null +++ b/src/libfetchers/tree-info.cc @@ -0,0 +1,14 @@ +#include "tree-info.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); +} + +} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh new file mode 100644 index 000000000..2c7347281 --- /dev/null +++ b/src/libfetchers/tree-info.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" + +#include <nlohmann/json_fwd.hpp> + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct TreeInfo +{ + Hash narHash; + std::optional<uint64_t> revCount; + std::optional<time_t> lastModified; + + bool operator ==(const TreeInfo & other) const + { + return + narHash == other.narHash + && revCount == other.revCount + && lastModified == other.lastModified; + } + + StorePath computeStorePath(Store & store) const; +}; + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9e1d7cee6..051668e53 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,48 +1,61 @@ #include "common-args.hh" #include "globals.hh" +#include "loggers.hh" namespace nix { MixCommonArgs::MixCommonArgs(const string & programName) : programName(programName) { - mkFlag() - .longName("verbose") - .shortName('v') - .description("increase verbosity level") - .handler([]() { verbosity = (Verbosity) (verbosity + 1); }); - - mkFlag() - .longName("quiet") - .description("decrease verbosity level") - .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); - - mkFlag() - .longName("debug") - .description("enable debug output") - .handler([]() { verbosity = lvlDebug; }); - - mkFlag() - .longName("option") - .labels({"name", "value"}) - .description("set a Nix configuration option (overriding nix.conf)") - .arity(2) - .handler([](std::vector<std::string> ss) { + addFlag({ + .longName = "verbose", + .shortName = 'v', + .description = "increase verbosity level", + .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, + }); + + addFlag({ + .longName = "quiet", + .description = "decrease verbosity level", + .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, + }); + + addFlag({ + .longName = "debug", + .description = "enable debug output", + .handler = {[]() { verbosity = lvlDebug; }}, + }); + + addFlag({ + .longName = "option", + .description = "set a Nix configuration option (overriding nix.conf)", + .labels = {"name", "value"}, + .handler = {[](std::string name, std::string value) { try { - globalConfig.set(ss[0], ss[1]); + globalConfig.set(name, value); } catch (UsageError & e) { warn(e.what()); } - }); - - mkFlag() - .longName("max-jobs") - .shortName('j') - .label("jobs") - .description("maximum number of parallel builds") - .handler([=](std::string s) { + }}, + }); + + addFlag({ + .longName = "log-format", + .description = "format of log output; \"raw\", \"internal-json\", \"bar\" " + "or \"bar-with-logs\"", + .labels = {"format"}, + .handler = {[](std::string format) { setLogFormat(format); }}, + }); + + addFlag({ + .longName = "max-jobs", + .shortName = 'j', + .description = "maximum number of parallel builds", + .labels = Strings{"jobs"}, + .handler = {[=](std::string s) { settings.set("max-jobs", s); - }); + }} + }); std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 0c80f5a0a..a8eed6c65 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -6,6 +6,8 @@ libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +libmain_CXXFLAGS += -I src/libutil -I src/libstore + libmain_LDFLAGS = $(OPENSSL_LIBS) libmain_LIBS = libstore libutil diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc new file mode 100644 index 000000000..c44bb6408 --- /dev/null +++ b/src/libmain/loggers.cc @@ -0,0 +1,52 @@ +#include "loggers.hh" +#include "progress-bar.hh" + +namespace nix { + +LogFormat defaultLogFormat = LogFormat::raw; + +LogFormat parseLogFormat(const std::string & logFormatStr) { + if (logFormatStr == "raw") + return LogFormat::raw; + else if (logFormatStr == "raw-with-logs") + return LogFormat::rawWithLogs; + else if (logFormatStr == "internal-json") + return LogFormat::internalJson; + else if (logFormatStr == "bar") + return LogFormat::bar; + else if (logFormatStr == "bar-with-logs") + return LogFormat::barWithLogs; + throw Error("option 'log-format' has an invalid value '%s'", logFormatStr); +} + +Logger * makeDefaultLogger() { + switch (defaultLogFormat) { + case LogFormat::raw: + return makeSimpleLogger(false); + case LogFormat::rawWithLogs: + return makeSimpleLogger(true); + case LogFormat::internalJson: + return makeJSONLogger(*makeSimpleLogger()); + case LogFormat::bar: + return makeProgressBar(); + case LogFormat::barWithLogs: + return makeProgressBar(true); + default: + abort(); + } +} + +void setLogFormat(const std::string & logFormatStr) { + setLogFormat(parseLogFormat(logFormatStr)); +} + +void setLogFormat(const LogFormat & logFormat) { + defaultLogFormat = logFormat; + createDefaultLogger(); +} + +void createDefaultLogger() { + logger = makeDefaultLogger(); +} + +} diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh new file mode 100644 index 000000000..cada03110 --- /dev/null +++ b/src/libmain/loggers.hh @@ -0,0 +1,20 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +enum class LogFormat { + raw, + rawWithLogs, + internalJson, + bar, + barWithLogs, +}; + +void setLogFormat(const std::string & logFormatStr); +void setLogFormat(const LogFormat & logFormat); + +void createDefaultLogger(); + +} diff --git a/src/nix/progress-bar.cc b/src/libmain/progress-bar.cc index 26631416c..95a9187de 100644 --- a/src/nix/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -7,6 +7,7 @@ #include <atomic> #include <map> #include <thread> +#include <iostream> namespace nix { @@ -105,25 +106,36 @@ public: updateThread.join(); } - void stop() + void stop() override { auto state(state_.lock()); if (!state->active) return; state->active = false; - std::string status = getStatus(*state); writeToStderr("\r\e[K"); - if (status != "") - writeToStderr("[" + status + "]\n"); updateCV.notify_one(); quitCV.notify_one(); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { auto state(state_.lock()); log(*state, lvl, fs.s); } + void logEI(const ErrorInfo &ei) override + { + auto state(state_.lock()); + + std::stringstream oss; + oss << ei; + + log(*state, ei.level, oss.str()); + } + void log(State & state, Verbosity lvl, const std::string & s) { if (state.active) { @@ -141,7 +153,7 @@ public: { auto state(state_.lock()); - if (lvl <= verbosity && !s.empty()) + if (lvl <= verbosity && !s.empty() && type != actBuildWaiting) log(*state, lvl, s + "..."); state->activities.emplace_back(ActInfo()); @@ -153,7 +165,7 @@ public: state->activitiesByType[type].its.emplace(act, i); if (type == actBuild) { - auto name = storePathToName(getS(fields, 0)); + std::string name(storePathToName(getS(fields, 0))); if (hasSuffix(name, ".drv")) name = name.substr(0, name.size() - 4); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); @@ -190,8 +202,8 @@ public: i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); } - if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) - || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) + if ((type == actFileTransfer && hasAncestor(*state, actCopyPath, parent)) + || (type == actFileTransfer && hasAncestor(*state, actQueryPathInfo, parent)) || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; @@ -416,7 +428,7 @@ public: if (!s2.empty()) { res += " ("; res += s2; res += ')'; } } - showActivity(actDownload, "%s MiB DL", "%.1f", MiB); + showActivity(actFileTransfer, "%s MiB DL", "%.1f", MiB); { auto s = renderActivity(actOptimiseStore, "%s paths optimised"); @@ -442,13 +454,31 @@ public: return res; } + + void writeToStdout(std::string_view s) override + { + auto state(state_.lock()); + if (state->active) { + std::cerr << "\r\e[K"; + Logger::writeToStdout(s); + draw(*state); + } else { + Logger::writeToStdout(s); + } + } }; -void startProgressBar(bool printBuildLogs) +Logger * makeProgressBar(bool printBuildLogs) { - logger = new ProgressBar( + return new ProgressBar( printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"); + isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + ); +} + +void startProgressBar(bool printBuildLogs) +{ + logger = makeProgressBar(printBuildLogs); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/libmain/progress-bar.hh index 4d61175c2..7f0dafecf 100644 --- a/src/nix/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -4,6 +4,8 @@ namespace nix { +Logger * makeProgressBar(bool printBuildLogs = false); + void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d41e772e9..dc6d5e413 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "util.hh" +#include "loggers.hh" #include <algorithm> #include <cctype> @@ -75,7 +76,7 @@ string getArg(const string & opt, Strings::iterator & i, const Strings::iterator & end) { ++i; - if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + if (i == end) throw UsageError("'%1%' requires an argument", opt); return *i; } @@ -155,7 +156,7 @@ void initNix() sshd). This breaks build users because they don't have access to the TMPDIR, in particular in ‘nix-store --serve’. */ #if __APPLE__ - if (getuid() == 0 && hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) unsetenv("TMPDIR"); #endif } @@ -165,28 +166,32 @@ LegacyArgs::LegacyArgs(const std::string & programName, std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) : MixCommonArgs(programName), parseArg(parseArg) { - mkFlag() - .longName("no-build-output") - .shortName('Q') - .description("do not show build output") - .set(&settings.verboseBuild, false); - - mkFlag() - .longName("keep-failed") - .shortName('K') - .description("keep temporary directories of failed builds") - .set(&(bool&) settings.keepFailed, true); - - mkFlag() - .longName("keep-going") - .shortName('k') - .description("keep going after a build fails") - .set(&(bool&) settings.keepGoing, true); - - mkFlag() - .longName("fallback") - .description("build from source if substitution fails") - .set(&(bool&) settings.tryFallback, true); + addFlag({ + .longName = "no-build-output", + .shortName = 'Q', + .description = "do not show build output", + .handler = {[&]() {setLogFormat(LogFormat::raw); }}, + }); + + addFlag({ + .longName = "keep-failed", + .shortName ='K', + .description = "keep temporary directories of failed builds", + .handler = {&(bool&) settings.keepFailed, true}, + }); + + addFlag({ + .longName = "keep-going", + .shortName ='k', + .description = "keep going after a build fails", + .handler = {&(bool&) settings.keepGoing, true}, + }); + + addFlag({ + .longName = "fallback", + .description = "build from source if substitution fails", + .handler = {&(bool&) settings.tryFallback, true}, + }); auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { @@ -205,11 +210,12 @@ LegacyArgs::LegacyArgs(const std::string & programName, mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", &gcWarning, false); - mkFlag() - .longName("store") - .label("store-uri") - .description("URI of the Nix store to use") - .dest(&(std::string&) settings.storeUri); + addFlag({ + .longName = "store", + .description = "URI of the Nix store to use", + .labels = {"store-uri"}, + .handler = {&(std::string&) settings.storeUri}, + }); } @@ -229,7 +235,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish) Strings ss(args); auto pos = ss.begin(); if (!parseArg(pos, ss.end())) - throw UsageError(format("unexpected argument '%1%'") % args.front()); + throw UsageError("unexpected argument '%1%'", args.front()); return true; } @@ -260,7 +266,10 @@ void printVersion(const string & programName) cfg.push_back("signed-caches"); #endif std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; - std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "System configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "User configuration files: " << + concatStringsSep(":", settings.nixUserConfFiles) + << "\n"; std::cout << "Store directory: " << settings.nixStore << "\n"; std::cout << "State directory: " << settings.nixStateDir << "\n"; } @@ -273,7 +282,7 @@ void showManPage(const string & name) restoreSignals(); setenv("MANPATH", settings.nixManDir.c_str(), 1); execlp("man", "man", name.c_str(), nullptr); - throw SysError(format("command 'man %1%' failed") % name.c_str()); + throw SysError("command 'man %1%' failed", name.c_str()); } @@ -281,6 +290,8 @@ int handleExceptions(const string & programName, std::function<void()> fun) { ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + ErrorInfo::programName = baseNameOf(programName); + string error = ANSI_RED "error:" ANSI_NORMAL " "; try { try { @@ -296,12 +307,13 @@ int handleExceptions(const string & programName, std::function<void()> fun) } catch (Exit & e) { return e.status; } catch (UsageError & e) { - printError( - format(error + "%1%\nTry '%2% --help' for more information.") - % e.what() % programName); + logError(e.info()); + printError("Try '%1% --help' for more information.", programName); return 1; } catch (BaseError & e) { - printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + if (settings.showTrace && e.prefix() != "") + printError(e.prefix()); + logError(e.info()); if (e.prefix() != "" && !settings.showTrace) printError("(use '--show-trace' to show detailed location information)"); return e.status; @@ -338,7 +350,7 @@ RunPager::RunPager() execlp("pager", "pager", nullptr); execlp("less", "less", nullptr); execlp("more", "more", nullptr); - throw SysError(format("executing '%1%'") % pager); + throw SysError("executing '%1%'", pager); }); pid.setKillSignal(SIGINT); diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index b49574652..f558247c0 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -56,7 +56,7 @@ template<class N> N getIntArg(const string & opt, Strings::iterator & i, const Strings::iterator & end, bool allowUnit) { ++i; - if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + if (i == end) throw UsageError("'%1%' requires an argument", opt); string s = *i; N multiplier = 1; if (allowUnit && !s.empty()) { @@ -66,13 +66,13 @@ template<class N> N getIntArg(const string & opt, else if (u == 'M') multiplier = 1ULL << 20; else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError(format("invalid unit specifier '%1%'") % u); + else throw UsageError("invalid unit specifier '%1%'", u); s.resize(s.size() - 1); } } N n; if (!string2Int(s, n)) - throw UsageError(format("'%1%' requires an integer argument") % opt); + throw UsageError("'%1%' requires an integer argument", opt); return n * multiplier; } diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index e6224de7d..b0a4a4c5d 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -1,4 +1,4 @@ -#include "types.hh" +#include "error.hh" #include <cstring> #include <cstddef> diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3a2d84861..f8eff508c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -40,14 +40,14 @@ void BinaryCacheStore::init() upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { - size_t colon = line.find(':'); - if (colon == std::string::npos) continue; + size_t colon= line.find(':'); + if (colon ==std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) - throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") - % getUri() % value % storeDir); + throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", + getUri(), value, storeDir); } else if (name == "WantMassQuery") { wantMassQuery.setDefault(value == "1" ? "true" : "false"); } else if (name == "Priority") { @@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - return storePathToHash(printStorePath(storePath)) + ".narinfo"; + return std::string(storePath.hashPart()) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) @@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(printStorePath(narInfo->path)); + std::string hashPart(narInfo->path.hashPart()); { auto state_(state.lock()); @@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { + // FIXME: See if we can use the original source to reduce memory usage. + auto nar = make_ref<std::string>(narSource.drain()); + if (!repair && isValidPath(info.path)) return; /* Verify that all references are valid. This may do some .narinfo @@ -161,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } } - upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ @@ -284,7 +287,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.what()); + throw SubstituteGone(e.info()); } decompressor->finish(); @@ -327,7 +330,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -336,7 +339,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { @@ -345,9 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, h, name)); - addToStore(info, sink.s, repair, CheckSigs, nullptr); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); return std::move(info.path); } @@ -356,12 +360,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { ValidPathInfo info(computeStorePathForText(name, s, references)); - info.references = cloneStorePathSet(references); + info.references = references; if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, sink.s, repair, CheckSigs, nullptr); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); } return std::move(info.path); @@ -390,14 +395,14 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) { - auto drvPath = path.clone(); + auto drvPath = path; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo if (!info->deriver) return nullptr; - drvPath = info->deriver->clone(); + drvPath = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index aa13c1cb4..52ef8aa7a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -74,12 +74,12 @@ public: std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 9c6aedfa5..53a0958aa 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -6,7 +6,8 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" -#include "download.hh" +#include "builtins/buildenv.hh" +#include "filetransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" @@ -32,7 +33,6 @@ #include <sys/types.h> #include <sys/stat.h> #include <sys/utsname.h> -#include <sys/select.h> #include <sys/resource.h> #include <sys/socket.h> #include <sys/un.h> @@ -42,6 +42,7 @@ #include <errno.h> #include <cstring> #include <termios.h> +#include <poll.h> #include <pwd.h> #include <grp.h> @@ -103,13 +104,10 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; -class Goal : public std::enable_shared_from_this<Goal> +struct Goal : public std::enable_shared_from_this<Goal> { -public: typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; -protected: - /* Backlink to the worker. */ Worker & worker; @@ -137,6 +135,9 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + /* Exception containing an error message, if any. */ + std::optional<Error> ex; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -148,7 +149,6 @@ protected: trace("goal destroyed"); } -public: virtual void work() = 0; void addWaitee(GoalPtr waitee); @@ -172,21 +172,14 @@ public: return name; } - ExitCode getExitCode() - { - return exitCode; - } - /* Callback in case of a timeout. It should wake up its waiters, get rid of any running child processes that are being monitored by the worker (important!), etc. */ - virtual void timedOut() = 0; + virtual void timedOut(Error && ex) = 0; virtual string key() = 0; -protected: - - virtual void amDone(ExitCode result); + void amDone(ExitCode result, std::optional<Error> ex = {}); }; @@ -302,7 +295,7 @@ public: /* Make a goal (with caching). */ GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath, + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); @@ -354,13 +347,13 @@ public: contents. */ bool pathContentsGood(const StorePath & path); - void markContentsGood(StorePath && path); + void markContentsGood(const StorePath & path); void updateProgress() { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; @@ -391,8 +384,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) assert(waitees.find(waitee) != waitees.end()); waitees.erase(waitee); - trace(format("waitee '%1%' done; %2% left") % - waitee->name % waitees.size()); + trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; @@ -417,12 +409,20 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } -void Goal::amDone(ExitCode result) +void Goal::amDone(ExitCode result, std::optional<Error> ex) { trace("done"); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; + + if (ex) { + if (!waiters.empty()) + logError(ex->info()); + else + this->ex = std::move(*ex); + } + for (auto & i : waiters) { GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); @@ -452,7 +452,7 @@ static void commonChildInit(Pipe & logPipe) that e.g. ssh cannot open /dev/tty) and it doesn't receive terminal signals. */ if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); /* Dup the write side of the logger pipe into stderr. */ if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) @@ -465,7 +465,7 @@ static void commonChildInit(Pipe & logPipe) /* Reroute stdin to /dev/null. */ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); if (fdDevNull == -1) - throw SysError(format("cannot open '%1%'") % pathNullDevice); + throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) throw SysError("cannot dup null device into stdin"); close(fdDevNull); @@ -487,12 +487,18 @@ void handleDiffHook( auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); + ErrorInfo ei = error.info(); + ei.hint = hintfmt("diff hook execution failed: %s", + (error.info().hint.has_value() ? error.info().hint->str() : "")); + logError(ei); } } } @@ -506,9 +512,10 @@ private: Path fnUserLock; AutoCloseFD fdUserLock; + bool isEnabled = false; string user; - uid_t uid; - gid_t gid; + uid_t uid = 0; + gid_t gid = 0; std::vector<gid_t> supplementaryGIDs; public: @@ -521,7 +528,9 @@ public: uid_t getGID() { assert(gid); return gid; } std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; } - bool enabled() { return uid != 0; } + bool findFreeUser(); + + bool enabled() { return isEnabled; } }; @@ -529,42 +538,46 @@ public: UserLock::UserLock() { assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); +} + +bool UserLock::findFreeUser() { + if (enabled()) return true; /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - throw Error(format("the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + throw Error("the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup); gid = gr->gr_gid; /* Copy the result of getgrnam. */ Strings users; for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user '%1%'") % *p); + debug("found build user '%1%'", *p); users.push_back(*p); } if (users.empty()) - throw Error(format("the build users group '%1%' has no members") - % settings.buildUsersGroup); + throw Error("the build users group '%1%' has no members", + settings.buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ for (auto & i : users) { - debug(format("trying user '%1%'") % i); + debug("trying user '%1%'", i); struct passwd * pw = getpwnam(i.c_str()); if (!pw) - throw Error(format("the user '%1%' in the group '%2%' does not exist") - % i % settings.buildUsersGroup); + throw Error("the user '%1%' in the group '%2%' does not exist", + i, settings.buildUsersGroup); - createDirs(settings.nixStateDir + "/userpool"); fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); + throw SysError("opening user lock '%1%'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { fdUserLock = std::move(fd); @@ -573,8 +586,8 @@ UserLock::UserLock() /* Sanity check... */ if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + throw Error("the Nix user should not be a member of '%1%'", + settings.buildUsersGroup); #if __linux__ /* Get the list of supplementary groups of this build user. This @@ -584,21 +597,19 @@ UserLock::UserLock() int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), &ngroups); if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); supplementaryGIDs.resize(ngroups); #endif - return; + isEnabled = true; + return true; } } - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the '%1%' group") - % settings.buildUsersGroup); + return false; } - void UserLock::kill() { killUser(uid); @@ -857,6 +868,9 @@ private: std::unique_ptr<Activity> act; + /* Activity that denotes waiting for a lock. */ + std::unique_ptr<Activity> actLock; + std::map<ActivityId, Activity> builderActivities; /* The remote machine on which we're building. */ @@ -886,16 +900,16 @@ private: friend struct RestrictedStore; public: - DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, + DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, + DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); - void timedOut() override; + void timedOut(Error && ex) override; string key() override { @@ -910,7 +924,7 @@ public: StorePath getDrvPath() { - return drvPath.clone(); + return drvPath; } /* Add wanted outputs to an already existing derivation goal. */ @@ -927,6 +941,7 @@ private: void closureRepaired(); void inputsRealised(); void tryToBuild(); + void tryLocalBuild(); void buildDone(); /* Is the build hook willing to perform the build? */ @@ -993,12 +1008,11 @@ private: void repairClosure(); - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + void started(); - void done(BuildResult::Status status, const string & msg = ""); + void done( + BuildResult::Status status, + std::optional<Error> ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); }; @@ -1007,11 +1021,11 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { @@ -1024,11 +1038,11 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut } -DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , buildMode(buildMode) { this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); @@ -1092,10 +1106,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::timedOut() +void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut); + done(BuildResult::TimedOut, ex); } @@ -1143,8 +1157,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)); - done(BuildResult::MiscFailure); + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -1180,7 +1193,7 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *drv); + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1288,14 +1301,18 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); + outputsToDrv.insert_or_assign(j.second.path, i); } /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError("found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + logError({ + .name = "Corrupt path in closure", + .hint = hintfmt( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) + }); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); @@ -1329,9 +1346,9 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - printError("cannot build derivation '%s': %s dependencies couldn't be built", - worker.store.printStorePath(drvPath), nrFailed); - done(BuildResult::DependencyFailed); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); return; } @@ -1385,6 +1402,19 @@ void DerivationGoal::inputsRealised() result = BuildResult(); } +void DerivationGoal::started() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); + worker.updateProgress(); +} void DerivationGoal::tryToBuild() { @@ -1397,13 +1427,18 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath))); + lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } + actLock.reset(); + /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After it has finished and released the locks, we can (and should) @@ -1419,7 +1454,7 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = cloneStorePathSet(drv->outputPaths()); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); @@ -1428,7 +1463,7 @@ void DerivationGoal::tryToBuild() for (auto & i : drv->outputs) { if (worker.store.isValidPath(i.second.path)) continue; debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path))); + deletePath(worker.store.Store::toRealPath(i.second.path)); } /* Don't do a remote build if the derivation has the attribute @@ -1436,26 +1471,13 @@ void DerivationGoal::tryToBuild() supported for local builds. */ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); - auto started = [&]() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); - worker.updateProgress(); - }; - /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + actLock.reset(); result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); @@ -1463,6 +1485,9 @@ void DerivationGoal::tryToBuild() case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1472,6 +1497,8 @@ void DerivationGoal::tryToBuild() } } + actLock.reset(); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1482,17 +1509,48 @@ void DerivationGoal::tryToBuild() return; } + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique<UserLock>(); + + if (buildUser->findFreeUser()) { + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } else { + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + return; + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + + actLock.reset(); + try { /* Okay, we have to build. */ startBuilder(); } catch (BuildError & e) { - printError(e.msg()); outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e.msg()); + done(BuildResult::InputRejected, e); return; } @@ -1604,10 +1662,10 @@ void DerivationGoal::buildDone() } auto msg = fmt("builder for '%s' %s", - worker.store.printStorePath(drvPath), + yellowtxt(worker.store.printStorePath(drvPath)), statusToString(status)); - if (!settings.verboseBuild && !logTail.empty()) { + if (!logger->isVerbose() && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); for (auto & line : logTail) msg += "\n " + line; @@ -1656,11 +1714,7 @@ void DerivationGoal::buildDone() } void flushLine() { - if (settings.verboseBuild) { - printError("post-build-hook: " + currentLine); - } else { - act.result(resPostBuildLogLine, currentLine); - } + act.result(resPostBuildLogLine, currentLine); currentLine.clear(); } @@ -1679,13 +1733,14 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { + deleteTmpDir(true); done(BuildResult::Built); return; } /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second))); + deletePath(worker.store.Store::toRealPath(i.second)); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -1708,8 +1763,6 @@ void DerivationGoal::buildDone() outputLocks.unlock(); } catch (BuildError & e) { - printError(e.msg()); - outputLocks.unlock(); BuildResult::Status st = BuildResult::MiscFailure; @@ -1728,7 +1781,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e.msg()); + done(st, e); return; } @@ -1771,7 +1824,7 @@ HookReply DerivationGoal::tryBuildHook() } } - debug(format("hook reply is '%1%'") % reply); + debug("hook reply is '%1%'", reply); if (reply == "decline") return rpDecline; @@ -1787,8 +1840,12 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - printError("build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); + logError({ + .name = "Build hook died", + .hint = hintfmt( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))) + }); worker.hook = 0; return rpDecline; } else @@ -1844,14 +1901,14 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (!inputPaths.count(storePath)) throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - worker.store.computeFSClosure(singleton(storePath), paths); + worker.store.computeFSClosure({storePath}, paths); } /* If there are derivations in the graph, then include their outputs as well. This is useful if you want to do things like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ - auto paths2 = cloneStorePathSet(paths); + auto paths2 = paths; for (auto & j : paths2) { if (j.isDerivation()) { @@ -1904,7 +1961,7 @@ void DerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", settings.systemFeatures)); + concatStringsSep<StringSet>(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); @@ -1941,22 +1998,6 @@ void DerivationGoal::startBuilder() #endif } - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - buildUser = std::make_unique<UserLock>(); - - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); @@ -1984,7 +2025,7 @@ void DerivationGoal::startBuilder() string s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); @@ -1996,7 +2037,7 @@ void DerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences(singleton(storePath)), false, false)); + exportReferences({storePath}), false, false)); } } @@ -2033,7 +2074,7 @@ void DerivationGoal::startBuilder() worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + throw Error("while processing 'sandbox-paths': %s", e.what()); } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -2071,7 +2112,7 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot"; + chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ @@ -2080,10 +2121,10 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError(format("cannot create '%1%'") % chrootRootDir); + throw SysError("cannot create '%1%'", chrootRootDir); if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir); + throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need this. (Of course they should really respect $TMPDIR @@ -2127,7 +2168,7 @@ void DerivationGoal::startBuilder() chmod_(chrootStoreDir, 01775); if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); @@ -2160,7 +2201,7 @@ void DerivationGoal::startBuilder() if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("directory '%1%' exists; please remove it") % homeDir); + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete @@ -2181,7 +2222,7 @@ void DerivationGoal::startBuilder() for (auto & i : missingPaths) if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i.clone()); + redirectedBadOutputs.insert(i); } } @@ -2205,8 +2246,7 @@ void DerivationGoal::startBuilder() if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { - throw Error(format("unknown pre-build hook command '%1%'") - % line); + throw Error("unknown pre-build hook command '%1%'", line); } } else if (state == stExtraChrootDirs) { if (line == "") { @@ -2228,7 +2268,7 @@ void DerivationGoal::startBuilder() startDaemon(); /* Run the builder. */ - printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2248,10 +2288,13 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); - } else { + } +#if __APPLE__ + else { if (grantpt(builderOut.readSide.get())) throw SysError("granting access to pseudoterminal slave"); } +#endif #if 0 // Mount the pt in the sandbox so that the "tty" command works. @@ -2464,7 +2507,7 @@ void DerivationGoal::initTmpDir() { auto hash = hashString(htSHA256, i.first); string fn = ".attr-" + hash.to_string(Base32, false); Path p = tmpDir + "/" + fn; - writeFile(p, i.second); + writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } @@ -2550,7 +2593,7 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + auto structuredAttrs = parsedDrv->getStructuredAttrs(); if (!structuredAttrs) return; auto json = *structuredAttrs; @@ -2674,8 +2717,8 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryAllValidPaths() override { StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p.clone()); - for (auto & p : goal.addedPaths) paths.insert(p.clone()); + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); return paths; } @@ -2704,14 +2747,11 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const StorePath & path) override - { throw Error("queryDerivationOutputNames"); } - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2724,9 +2764,9 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { - auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); return path; } @@ -2766,7 +2806,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path.clone()); + newPaths.insert(output.second.path); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -2811,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore if (goal.isAllowed(path.path)) allowed.emplace_back(path); else - unknown.insert(path.path.clone()); + unknown.insert(path.path); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -2906,7 +2946,7 @@ void DerivationGoal::addDependency(const StorePath & path) { if (isAllowed(path)) return; - addedPaths.insert(path.clone()); + addedPaths.insert(path); /* If we're doing a sandbox build, then we have to make the path appear in the sandbox. */ @@ -2916,7 +2956,7 @@ void DerivationGoal::addDependency(const StorePath & path) #if __linux__ - Path source = worker.store.toRealPath(worker.store.printStorePath(path)); + Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); debug("bind-mounting %s -> %s", target, source); @@ -2966,7 +3006,7 @@ void DerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); + throw SysError("cannot change ownership of '%1%'", path); } @@ -3103,7 +3143,7 @@ void DerivationGoal::runChild() /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + throw SysError("unable to bind mount '%1%'", chrootRootDir); /* Bind-mount the sandbox's Nix store onto itself so that we can mark it as a "shared" subtree, allowing bind @@ -3150,7 +3190,7 @@ void DerivationGoal::runChild() // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the - // potential impurities introduced in fixed outputs. + // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); ss.push_back("/etc/services"); @@ -3165,7 +3205,7 @@ void DerivationGoal::runChild() filesystem that we want in the chroot environment. */ auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug(format("bind mounting '%1%' to '%2%'") % source % target); + debug("bind mounting '%1%' to '%2%'", source, target); struct stat st; if (stat(source.c_str(), &st) == -1) { if (optional && errno == ENOENT) @@ -3237,16 +3277,16 @@ void DerivationGoal::runChild() /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); + throw SysError("cannot change directory to '%1%'", chrootRootDir); if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) - throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root")); + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); if (chroot(".") == -1) - throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir); + throw SysError("cannot change root directory to '%1%'", chrootRootDir); if (umount2("real-root", MNT_DETACH) == -1) throw SysError("cannot unmount real root filesystem"); @@ -3267,7 +3307,7 @@ void DerivationGoal::runChild() #endif if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError(format("changing into '%1%'") % tmpDir); + throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); @@ -3406,9 +3446,9 @@ void DerivationGoal::runChild() sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second.source) - throw Error(format( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second.source); + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); string path = i.first; struct stat st; @@ -3499,7 +3539,7 @@ void DerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); @@ -3509,7 +3549,7 @@ void DerivationGoal::runChild() execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing '%1%'") % drv->builder); + throw SysError("executing '%1%'", drv->builder); } catch (std::exception & e) { writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); @@ -3528,13 +3568,36 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path.clone()); + result.insert(drv.outputs.find(i)->second.path); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; } +static void moveCheckToStore(const Path & src, const Path & dst) +{ + /* For the rename of directory to succeed, we must be running as root or + the directory must be made temporarily writable (to update the + directory's parent link ".."). */ + struct stat st; + if (lstat(src.c_str(), &st) == -1) { + throw SysError("getting attributes of path '%1%'", src); + } + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output @@ -3563,9 +3626,9 @@ void DerivationGoal::registerOutputs() output paths, and any paths that have been built via recursive Nix calls. */ StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); - for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -3578,7 +3641,7 @@ void DerivationGoal::registerOutputs() if (needsHashRewrite()) { auto r = redirectedOutputs.find(i.second.path); if (r != redirectedOutputs.end()) { - auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second)); + auto redirected = worker.store.Store::toRealPath(r->second); if (buildMode == bmRepair && redirectedBadOutputs.count(i.second.path) && pathExists(redirected)) @@ -3594,7 +3657,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); + throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); } @@ -3615,13 +3678,16 @@ void DerivationGoal::registerOutputs() user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path); + throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); #endif /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path); + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) + }); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -3646,21 +3712,25 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - bool recursive; Hash h; - i.second.parseHashInfo(recursive, h); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); - if (!recursive) { + if (outputHashMode == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path '%1%' should be a non-executable regular file") % path); + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat)", + path); } /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + Hash h2 = outputHashMode == FileIngestionMethod::Recursive + ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); if (h != h2) { @@ -3669,9 +3739,9 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); - Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); + Path actualDest = worker.store.Store::toRealPath(dest); if (worker.store.isValidPath(dest)) std::rethrow_exception(delayedException); @@ -3689,7 +3759,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(outputHashMode, h2); } /* Get rid of all weird permissions. This also checks that @@ -3713,8 +3783,7 @@ void DerivationGoal::registerOutputs() if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); - if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + moveCheckToStore(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), @@ -3722,10 +3791,10 @@ void DerivationGoal::registerOutputs() path, dst, worker.store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), path, dst); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), path); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); } /* Since we verified the build, it's now ultimately trusted. */ @@ -3758,7 +3827,7 @@ void DerivationGoal::registerOutputs() info.narHash = hash.first; info.narSize = hash.second; info.references = std::move(references); - info.deriver = drvPath.clone(); + info.deriver = drvPath; info.ultimate = true; info.ca = ca; worker.store.signPathInfo(info); @@ -3782,10 +3851,10 @@ void DerivationGoal::registerOutputs() result.isNonDeterministic = true; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); - auto msg = prevExists - ? fmt("output '%s' of '%s' differs from '%s' from previous round", + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : fmt("output '%s' of '%s' differs from previous round", + : hintfmt("output '%s' of '%s' differs from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); handleDiffHook( @@ -3795,9 +3864,13 @@ void DerivationGoal::registerOutputs() worker.store.printStorePath(drvPath), tmpDir); if (settings.enforceDeterminism) - throw NotDeterministic(msg); + throw NotDeterministic(hint); + + logError({ + .name = "Output determinism error", + .hint = hint + }); - printError(msg); curRound = nrRounds; // we know enough, bail out early } } @@ -3869,23 +3942,23 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) uint64_t closureSize = 0; StorePathSet pathsDone; std::queue<StorePath> pathsLeft; - pathsLeft.push(path.clone()); + pathsLeft.push(path); while (!pathsLeft.empty()) { - auto path = pathsLeft.front().clone(); + auto path = pathsLeft.front(); pathsLeft.pop(); - if (!pathsDone.insert(path.clone()).second) continue; + if (!pathsDone.insert(path).second) continue; auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } } @@ -3911,7 +3984,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); + auto used = recursive + ? getClosure(info.path).first + : info.references; if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -3921,10 +3996,10 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) for (auto & i : used) if (allowed) { if (!spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } else { if (spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } if (!badPaths.empty()) { @@ -4013,7 +4088,7 @@ Path DerivationGoal::openLogFile() settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared<FdSink>(fdLogFile.get()); @@ -4059,11 +4134,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - printError( - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); killChild(); - done(BuildResult::LogLimitExceeded); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); return; } @@ -4105,13 +4180,8 @@ void DerivationGoal::flushLine() ; else { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(currentLogLine); - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - } + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); act->result(resBuildLogLine, currentLogLine); } @@ -4129,7 +4199,7 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path.clone()); + if (good == returnValid) result.insert(i.second.path); } return result; } @@ -4145,15 +4215,16 @@ void DerivationGoal::addHashRewrite(const StorePath & path) deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); + redirectedOutputs.insert_or_assign(path, std::move(p)); } -void DerivationGoal::done(BuildResult::Status status, const string & msg) +void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) { result.status = status; - result.errorMsg = msg; - amDone(result.success() ? ecSuccess : ecFailed); + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); if (result.status == BuildResult::TimedOut) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) @@ -4219,10 +4290,10 @@ private: GoalState state; public: - SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); - void timedOut() override { abort(); }; + void timedOut(Error && ex) override { abort(); }; string key() override { @@ -4245,18 +4316,13 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - StorePath getStorePath() { return storePath.clone(); } - - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + StorePath getStorePath() { return storePath; } }; -SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) : Goal(worker) - , storePath(std::move(storePath)) + , storePath(storePath) , repair(repair) { state = &SubstitutionGoal::init; @@ -4351,7 +4417,7 @@ void SubstitutionGoal::tryNext() throw; } catch (Error & e) { if (settings.tryFallback) { - printError(e.what()); + logError(e.info()); tryNext(); return; } @@ -4377,8 +4443,11 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)); + logWarning({ + .name = "Invalid path signature", + .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)) + }); tryNext(); return; } @@ -4487,7 +4556,7 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath.clone()); + worker.markContentsGood(storePath); printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); @@ -4521,7 +4590,6 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -4558,10 +4626,10 @@ Worker::~Worker() GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME + GoalPtr goal = derivationGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared<DerivationGoal>(path.clone(), wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path, goal); wakeUp(goal); } else (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); @@ -4569,10 +4637,10 @@ GoalPtr Worker::makeDerivationGoal(const StorePath & path, } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drvPath, +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(std::move(drvPath), drv, *this, buildMode); + auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); wakeUp(goal); return goal; } @@ -4580,10 +4648,10 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drv GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME + GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair); - substitutionGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair); + substitutionGoals.insert_or_assign(path, goal); wakeUp(goal); } return goal; @@ -4612,7 +4680,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } @@ -4738,9 +4806,9 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( - "unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + if (awake.empty() && 0 == settings.maxBuildJobs) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); assert(!awake.empty()); } } @@ -4753,7 +4821,6 @@ void Worker::run(const Goals & _topGoals) assert(!settings.keepGoing || children.empty()); } - void Worker::waitForInput() { printMsg(lvlVomit, "waiting for children"); @@ -4765,8 +4832,7 @@ void Worker::waitForInput() terminated. */ bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; + long timeout = 0; auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there @@ -4784,7 +4850,7 @@ void Worker::waitForInput() nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { - timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); + timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); useTimeout = true; } @@ -4792,33 +4858,29 @@ void Worker::waitForInput() up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks or build slots..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max(1L, + timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>( lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); } else lastWokenUp = steady_time_point::min(); if (useTimeout) - vomit("sleeping %d seconds", timeout.tv_sec); + vomit("sleeping %d seconds", timeout); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; + std::vector<struct pollfd> pollStatus; + std::map <int, int> fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { - if (j >= FD_SETSIZE) - throw Error("reached FD_SETSIZE limit"); - FD_SET(j, &fds); - if (j >= fdMax) fdMax = j + 1; + pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + fdToPollStatus[j] = pollStatus.size() - 1; } } - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (poll(pollStatus.data(), pollStatus.size(), + useTimeout ? timeout * 1000 : -1) == -1) { if (errno == EINTR) return; throw SysError("waiting for input"); } @@ -4839,20 +4901,20 @@ void Worker::waitForInput() set<int> fds2(j->fds); std::vector<unsigned char> buffer(4096); for (auto & k : fds2) { - if (FD_ISSET(k, &fds)) { + if (pollStatus.at(fdToPollStatus.at(k)).revents) { ssize_t rd = read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { - debug(format("%1%: got EOF") % goal->getName()); + debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); j->fds.erase(k); } else if (rd == -1) { if (errno != EINTR) throw SysError("%s: read failed", goal->getName()); } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->getName() % rd); + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), rd); string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); @@ -4860,26 +4922,24 @@ void Worker::waitForInput() } } - if (goal->getExitCode() == Goal::ecBusy && + if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - printError( - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds of silence", + goal->getName(), settings.maxSilentTime)); } - else if (goal->getExitCode() == Goal::ecBusy && + else if (goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - printError( - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds", + goal->getName(), settings.buildTimeout)); } } @@ -4936,15 +4996,19 @@ bool Worker::pathContentsGood(const StorePath & path) Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache.insert_or_assign(path.clone(), res); - if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path)); + pathContentsGoodCache.insert_or_assign(path, res); + if (!res) + logError({ + .name = "Corrupted path", + .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) + }); return res; } -void Worker::markContentsGood(StorePath && path) +void Worker::markContentsGood(const StorePath & path) { - pathContentsGoodCache.insert_or_assign(std::move(path), true); + pathContentsGoodCache.insert_or_assign(path, true); } @@ -4981,24 +5045,35 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, worker.run(goals); StorePathSet failed; + std::optional<Error> ex; for (auto & i : goals) { - if (i->getExitCode() != Goal::ecSuccess) { + if (i->ex) { + if (ex) + logError(i->ex->info()); + else + ex = i->ex; + } + if (i->exitCode != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } } - if (!failed.empty()) + if (failed.size() == 1 && ex) { + ex->status = worker.exitStatus(); + throw *ex; + } else if (!failed.empty()) { + if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + } } - BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); BuildResult result; @@ -5019,7 +5094,7 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {StorePathWithOutputs(path)}); + primeCache(*this, {{path}}); Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); @@ -5027,8 +5102,13 @@ void LocalStore::ensurePath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + if (goal->exitCode != Goal::ecSuccess) { + if (goal->ex) { + goal->ex->status = worker.exitStatus(); + throw *goal->ex; + } else + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } } @@ -5040,7 +5120,7 @@ void LocalStore::repairPath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) { + if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 87d6ce665..66597e456 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,6 @@ namespace nix { // TODO: make pluggable. void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinBuildenv(const BasicDerivation & drv); void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 096593886..802fb87bc 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,4 @@ -#include "builtins.hh" +#include "buildenv.hh" #include <sys/stat.h> #include <sys/types.h> @@ -7,16 +7,14 @@ namespace nix { -typedef std::map<Path,int> Priorities; - -// FIXME: change into local variables. - -static Priorities priorities; - -static unsigned long symlinks; +struct State +{ + std::map<Path, int> priorities; + unsigned long symlinks = 0; +}; /* For each activated package, create symlinks */ -static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +static void createLinks(State & state, const Path & srcDir, const Path & dstDir, int priority) { DirEntries srcFiles; @@ -24,7 +22,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); + logWarning({ + .name = "Create links - directory", + .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) + }); return; } throw; @@ -43,7 +44,10 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - printError("warning: skipping dangling symlink '%s'", dstFile); + logWarning({ + .name = "Create links - skipping symlink", + .hint = hintfmt("skipping dangling symlink '%s'", dstFile) + }); continue; } throw; @@ -67,22 +71,22 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISDIR(dstSt.st_mode)) { - createLinks(srcFile, dstFile, priority); + createLinks(state, srcFile, dstFile, priority); continue; } else if (S_ISLNK(dstSt.st_mode)) { auto target = canonPath(dstFile, true); if (!S_ISDIR(lstat(target).st_mode)) throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory '%1%'")); - createLinks(target, dstFile, priorities[dstFile]); - createLinks(srcFile, dstFile, priority); + throw SysError("creating directory '%1%'", dstFile); + createLinks(state, target, dstFile, state.priorities[dstFile]); + createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } else { @@ -90,7 +94,7 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) auto res = lstat(dstFile.c_str(), &dstSt); if (res == 0) { if (S_ISLNK(dstSt.st_mode)) { - auto prevPriority = priorities[dstFile]; + auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( "packages '%1%' and '%2%' have the same priority %3%; " @@ -101,49 +105,65 @@ static void createLinks(const Path & srcDir, const Path & dstDir, int priority) if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } createSymlink(srcFile, dstFile); - priorities[dstFile] = priority; - symlinks++; + state.priorities[dstFile] = priority; + state.symlinks++; } } -typedef std::set<Path> FileProp; +void buildProfile(const Path & out, Packages && pkgs) +{ + State state; -static FileProp done; -static FileProp postponed = FileProp{}; + std::set<Path> done, postponed; -static Path out; + auto addPkg = [&](const Path & pkgDir, int priority) { + if (!done.insert(pkgDir).second) return; + createLinks(state, pkgDir, out, priority); -static void addPkg(const Path & pkgDir, int priority) -{ - if (!done.insert(pkgDir).second) return; - createLinks(pkgDir, out, priority); + try { + for (const auto & p : tokenizeString<std::vector<string>>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; + } + }; - try { - for (const auto & p : tokenizeString<std::vector<string>>( - readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) - if (!done.count(p)) - postponed.insert(p); - } catch (SysError & e) { - if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; - } -} + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { + return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); + }); + for (const auto & pkg : pkgs) + if (pkg.active) + addPkg(pkg.path, pkg.priority); -struct Package { - Path path; - bool active; - int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} -}; + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + std::set<Path> pkgDirs; + postponed.swap(pkgDirs); + for (const auto & pkgDir : pkgDirs) + addPkg(pkgDir, priorityCounter++); + } -typedef std::vector<Package> Packages; + debug("created %d symlinks in user environment", state.symlinks); +} void builtinBuildenv(const BasicDerivation & drv) { @@ -153,7 +173,7 @@ void builtinBuildenv(const BasicDerivation & drv) return i->second; }; - out = getAttr("out"); + Path out = getAttr("out"); createDirs(out); /* Convert the stuff we get from the environment back into a @@ -171,31 +191,7 @@ void builtinBuildenv(const BasicDerivation & drv) } } - /* Symlink to the packages that have been installed explicitly by the - * user. Process in priority order to reduce unnecessary - * symlink/unlink steps. - */ - std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { - return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); - }); - for (const auto & pkg : pkgs) - if (pkg.active) - addPkg(pkg.path, pkg.priority); - - /* Symlink to the packages that have been "propagated" by packages - * installed by the user (i.e., package X declares that it wants Y - * installed as well). We do these later because they have a lower - * priority in case of collisions. - */ - auto priorityCounter = 1000; - while (!postponed.empty()) { - auto pkgDirs = postponed; - postponed = FileProp{}; - for (const auto & pkgDir : pkgDirs) - addPkg(pkgDir, priorityCounter++); - } - - printError("created %d symlinks in user environment", symlinks); + buildProfile(out, std::move(pkgs)); createSymlink(getAttr("manifest"), out + "/manifest.nix"); } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh new file mode 100644 index 000000000..0a37459b0 --- /dev/null +++ b/src/libstore/builtins/buildenv.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "derivations.hh" +#include "store-api.hh" + +namespace nix { + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector<Package> Packages; + +void buildProfile(const Path & out, Packages && pkgs); + +void builtinBuildenv(const BasicDerivation & drv); + +} diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index f6ae5d2e6..2048f8f87 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" @@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto getAttr = [&](const string & name) { auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh downloader here because we're in + /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ - auto downloader = makeDownloader(); + auto fileTransfer = makeFileTransfer(); auto fetch = [&](const std::string & url) { @@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DownloadRequest request(url); + FileTransferRequest request(url); request.verifyTLS = false; request.decompress = false; auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - downloader->download(std::move(request), *decompressor); + fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); @@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto executable = drv.env.find("executable"); if (executable != drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); + throw SysError("making '%1%' executable", storePath); } }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8e9f9d71b..e370e278c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -73,6 +73,18 @@ struct TunnelLogger : public Logger enqueueMsg(*buf.s); } + void logEI(const ErrorInfo & ei) override + { + if (ei.level > verbosity) return; + + std::stringstream oss; + oss << ei; + + StringSink buf; + buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n"); + enqueueMsg(*buf.s); + } + /* startWork() means that we're starting an operation for which we want to send out stderr to the client. */ void startWork() @@ -281,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; // FIXME - paths.insert(path.clone()); + paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); to << (res.count(path) != 0); @@ -315,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, StorePathSet paths; if (op == wopQueryReferences) for (auto & i : store->queryPathInfo(path)->references) - paths.insert(i.clone()); + paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) @@ -329,8 +341,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); + auto names = store->readDerivation(path).outputNames(); logger->stopWork(); to << names; break; @@ -355,20 +366,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - bool fixed, recursive; std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; + FileIngestionMethod method; + { + bool fixed; uint8_t recursive; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + method = FileIngestionMethod::Recursive; + } } HashType hashAlgo = parseHashType(s); TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ @@ -380,7 +397,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + auto path = store->addToStoreFromDump( + method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + baseName, + method, + hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -572,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - StorePathSet paths; - paths.insert(path.clone()); // FIXME - store->querySubstitutablePathInfos(paths, infos); + store->querySubstitutablePathInfos({path}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -735,7 +754,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } default: - throw Error(format("invalid operation %1%") % op); + throw Error("invalid operation %1%", op); } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 4c51bdef3..065702c37 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,13 +9,13 @@ namespace nix { -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const { - recursive = false; + recursive = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = true; + recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); } @@ -27,28 +27,6 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const } -BasicDerivation::BasicDerivation(const BasicDerivation & other) - : platform(other.platform) - , builder(other.builder) - , args(other.args) - , env(other.env) -{ - for (auto & i : other.outputs) - outputs.insert_or_assign(i.first, - DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); - for (auto & i : other.inputSrcs) - inputSrcs.insert(i.clone()); -} - - -Derivation::Derivation(const Derivation & other) - : BasicDerivation(other) -{ - for (auto & i : other.inputDrvs) - inputDrvs.insert_or_assign(i.first.clone(), i.second); -} - - const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); @@ -65,16 +43,16 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref<Store> store, - const Derivation & drv, const string & name, RepairFlag repair) + const Derivation & drv, std::string_view name, RepairFlag repair) { - auto references = cloneStorePathSet(drv.inputSrcs); + auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) - references.insert(i.first.clone()); + references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ - string suffix = name + drvExtension; - string contents = drv.unparse(*store, false); + auto suffix = std::string(name) + drvExtension; + auto contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); @@ -87,7 +65,7 @@ static void expect(std::istream & str, const string & s) char s2[s.size()]; str.read(s2, s.size()); if (string(s2, s.size()) != s) - throw FormatError(format("expected string '%1%'") % s); + throw FormatError("expected string '%1%'", s); } @@ -114,7 +92,7 @@ static Path parsePath(std::istream & str) { string s = parseString(str); if (s.size() == 0 || s[0] != '/') - throw FormatError(format("bad path '%1%' in derivation") % s); + throw FormatError("bad path '%1%' in derivation", s); return s; } @@ -155,7 +133,11 @@ static Derivation parseDerivation(const Store & store, const string & s) expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hash = parseString(str); expect(str, ")"); - drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(id, DerivationOutput { + .path = std::move(path), + .hashAlgo = std::move(hashAlgo), + .hash = std::move(hash) + }); } /* Parse the list of input derivations. */ @@ -196,7 +178,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath) try { return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } } @@ -204,6 +186,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ auto accessor = getFSAccessor(); try { return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); @@ -351,12 +339,10 @@ static DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drv assert(store.isValidPath(drvPath)); // Cache it h = drvHashes.insert_or_assign( - drvPath.clone(), + drvPath, hashDerivationModulo( store, - readDerivation( - store, - store.toRealPath(store.printStorePath(drvPath))), + store.readDerivation(drvPath), false)).first; } return h->second; @@ -437,11 +423,20 @@ StorePathSet BasicDerivation::outputPaths() const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path.clone()); + paths.insert(i.second.path); return paths; } +StringSet BasicDerivation::outputNames() const +{ + StringSet names; + for (auto & i : outputs) + names.insert(i.first); + return names; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); @@ -451,7 +446,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) auto path = store.parseStorePath(readString(in)); auto hashAlgo = readString(in); auto hash = readString(in); - drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(name, DerivationOutput { + .path = std::move(path), + .hashAlgo = std::move(hashAlgo), + .hash = std::move(hash) + }); } drv.inputSrcs = readStorePaths<StorePathSet>(store, in); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 5e708642e..97271ac09 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -18,12 +18,7 @@ struct DerivationOutput StorePath path; std::string hashAlgo; /* hash used for expected hash computation */ std::string hash; /* expected hash, may be null */ - DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) - : path(std::move(path)) - , hashAlgo(std::move(hashAlgo)) - , hash(std::move(hash)) - { } - void parseHashInfo(bool & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -44,7 +39,6 @@ struct BasicDerivation StringPairs env; BasicDerivation() { } - explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in @@ -59,6 +53,8 @@ struct BasicDerivation /* Return the output paths of a derivation. */ StorePathSet outputPaths() const; + /* Return the output names of a derivation. */ + StringSet outputNames() const; }; struct Derivation : BasicDerivation @@ -70,8 +66,6 @@ struct Derivation : BasicDerivation std::map<std::string, StringSet> * actualInputs = nullptr) const; Derivation() { } - Derivation(Derivation && other) = default; - explicit Derivation(const Derivation & other); }; @@ -80,7 +74,7 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref<Store> store, - const Derivation & drv, const string & name, RepairFlag repair = NoRepair); + const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ Derivation readDerivation(const Store & store, const Path & drvPath); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4692d1a7b..cb9da027d 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,3 +1,4 @@ +#include "serialise.hh" #include "store-api.hh" #include "archive.hh" #include "worker-protocol.hh" @@ -56,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(), hash.to_string()); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic @@ -100,9 +101,11 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (readInt(source) == 1) readString(source); - addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); + // Can't use underlying source, which would have been exhausted + auto source = StringSource { *tee.source.data }; + addToStore(info, source, NoRepair, checkSigs, accessor); - res.push_back(info.path.clone()); + res.push_back(info.path); } return res; diff --git a/src/libstore/download.cc b/src/libstore/filetransfer.cc index 149c84765..c954ace7f 100644 --- a/src/libstore/download.cc +++ b/src/libstore/filetransfer.cc @@ -1,14 +1,10 @@ -#include "download.hh" +#include "filetransfer.hh" #include "util.hh" #include "globals.hh" -#include "hash.hh" #include "store-api.hh" -#include "archive.hh" #include "s3.hh" #include "compression.hh" -#include "pathlocks.hh" #include "finally.hh" -#include "tarfile.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -31,13 +27,9 @@ using namespace std::string_literals; namespace nix { -DownloadSettings downloadSettings; +FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&downloadSettings); - -CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) - : uri(uri), ttl(settings.tarballTtl) -{ } +static GlobalConfig::Register r1(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri) return uri; } -struct CurlDownloader : public Downloader +struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; std::random_device rd; std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this<DownloadItem> + struct TransferItem : public std::enable_shared_from_this<TransferItem> { - CurlDownloader & downloader; - DownloadRequest request; - DownloadResult result; + curlFileTransfer & fileTransfer; + FileTransferRequest request; + FileTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback<DownloadResult> callback; + Callback<FileTransferResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -80,19 +72,36 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; - DownloadItem(CurlDownloader & downloader, - const DownloadRequest & request, - Callback<DownloadResult> && callback) - : downloader(downloader) + /* Get the HTTP status code, or 0 for other protocols. */ + long getHTTPStatus() + { + long httpStatus = 0; + long protocol = 0; + curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol); + if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS) + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + return httpStatus; + } + + TransferItem(curlFileTransfer & fileTransfer, + const FileTransferRequest & request, + Callback<FileTransferResult> && callback) + : fileTransfer(fileTransfer) , request(request) - , act(*logger, lvlTalkative, actDownload, + , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + auto httpStatus = getHTTPStatus(); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char *) data, len); + } } else this->result.data->append((char *) data, len); }) @@ -103,17 +112,17 @@ struct CurlDownloader : public Downloader requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); } - ~DownloadItem() + ~TransferItem() { if (req) { if (active) - curl_multi_remove_handle(downloader.curlm, req); + curl_multi_remove_handle(fileTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -157,7 +166,7 @@ struct CurlDownloader : public Downloader static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); + return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } size_t headerCallback(void * contents, size_t size, size_t nmemb) @@ -199,7 +208,7 @@ struct CurlDownloader : public Downloader static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); + return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); } int progressCallback(double dltotal, double dlnow) @@ -214,7 +223,7 @@ struct CurlDownloader : public Downloader static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); + return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -238,7 +247,7 @@ struct CurlDownloader : public Downloader static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { - return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } void init() @@ -249,7 +258,7 @@ struct CurlDownloader : public Downloader if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); @@ -258,19 +267,19 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloadSettings.enableHttp2) + if (fileTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); @@ -298,10 +307,10 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -317,8 +326,7 @@ struct CurlDownloader : public Downloader void finish(CURLcode code) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + auto httpStatus = getHTTPStatus(); char * effectiveUriCStr; curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); @@ -345,7 +353,7 @@ struct CurlDownloader : public Downloader failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); @@ -390,6 +398,7 @@ struct CurlDownloader : public Downloader case CURLE_SSL_CACERT_BADFILE: case CURLE_TOO_MANY_REDIRECTS: case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: err = Misc; break; default: // Shut up warnings @@ -401,14 +410,14 @@ struct CurlDownloader : public Downloader auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DownloadError(err, + ? FileTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DownloadError(err, + : FileTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -422,13 +431,13 @@ struct CurlDownloader : public Downloader || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); + fileTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -439,12 +448,12 @@ struct CurlDownloader : public Downloader struct State { struct EmbargoComparator { - bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) { + bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) { return i1->embargo > i2->embargo; } }; bool quit = false; - std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming; + std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming; }; Sync<State> state_; @@ -456,7 +465,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; - CurlDownloader() + curlFileTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -469,7 +478,7 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - downloadSettings.httpConnections.get()); + fileTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); @@ -478,7 +487,7 @@ struct CurlDownloader : public Downloader workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~CurlDownloader() + ~curlFileTransfer() { stopWorkerThread(); @@ -504,7 +513,7 @@ struct CurlDownloader : public Downloader stopWorkerThread(); }); - std::map<CURL *, std::shared_ptr<DownloadItem>> items; + std::map<CURL *, std::shared_ptr<TransferItem>> items; bool quit = false; @@ -517,7 +526,7 @@ struct CurlDownloader : public Downloader int running; CURLMcode mc = curl_multi_perform(curlm, &running); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc)); /* Set the promises of any finished requests. */ CURLMsg * msg; @@ -547,7 +556,7 @@ struct CurlDownloader : public Downloader vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc)); nextWakeup = std::chrono::steady_clock::time_point(); @@ -561,7 +570,7 @@ struct CurlDownloader : public Downloader throw SysError("reading curl wakeup socket"); } - std::vector<std::shared_ptr<DownloadItem>> incoming; + std::vector<std::shared_ptr<TransferItem>> incoming; auto now = std::chrono::steady_clock::now(); { @@ -599,7 +608,11 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError("unexpected error in download thread: %s", e.what()); + logError({ + .name = "File transfer", + .hint = hintfmt("unexpected error in download thread: %s", + e.what()) + }); } { @@ -609,7 +622,7 @@ struct CurlDownloader : public Downloader } } - void enqueueItem(std::shared_ptr<DownloadItem> item) + void enqueueItem(std::shared_ptr<TransferItem> item) { if (item->request.data && !hasPrefix(item->request.uri, "http://") @@ -641,8 +654,8 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DownloadRequest & request, - Callback<DownloadResult> callback) override + void enqueueFileTransfer(const FileTransferRequest & request, + Callback<FileTransferResult> callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,9 +673,9 @@ struct CurlDownloader : public Downloader // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DownloadResult res; + FileTransferResult res; if (!s3Res.data) - throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else @@ -672,26 +685,26 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); + enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback))); } }; -ref<Downloader> getDownloader() +ref<FileTransfer> getFileTransfer() { - static ref<Downloader> downloader = makeDownloader(); - return downloader; + static ref<FileTransfer> fileTransfer = makeFileTransfer(); + return fileTransfer; } -ref<Downloader> makeDownloader() +ref<FileTransfer> makeFileTransfer() { - return make_ref<CurlDownloader>(); + return make_ref<curlFileTransfer>(); } -std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request) +std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) { - auto promise = std::make_shared<std::promise<DownloadResult>>(); - enqueueDownload(request, - {[promise](std::future<DownloadResult> fut) { + auto promise = std::make_shared<std::promise<FileTransferResult>>(); + enqueueFileTransfer(request, + {[promise](std::future<FileTransferResult> fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,15 +714,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & return promise->get_future(); } -DownloadResult Downloader::download(const DownloadRequest & request) +FileTransferResult FileTransfer::download(const FileTransferRequest & request) +{ + return enqueueFileTransfer(request).get(); +} + +FileTransferResult FileTransfer::upload(const FileTransferRequest & request) { - return enqueueDownload(request).get(); + /* Note: this method is the same as download, but helps in readability */ + return enqueueFileTransfer(request).get(); } -void Downloader::download(DownloadRequest && request, Sink & sink) +void FileTransfer::download(FileTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the downloader + that would cause the sink to execute on the fileTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. @@ -755,8 +774,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDownload(request, - {[_state](std::future<DownloadResult> fut) { + enqueueFileTransfer(request, + {[_state](std::future<FileTransferResult> fut) { auto state(_state->lock()); state->quit = true; try { @@ -801,140 +820,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached( - ref<Store> store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - std::optional<StorePath> expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(*expectedStorePath)) { - CachedDownloadResult result; - result.storePath = store->printStorePath(*expectedStorePath); - result.path = store->toRealPath(result.storePath); - return result; - } - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - - std::optional<StorePath> storePath; - - string expectedETag; - - bool skip = false; - - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = store->parseStorePath(readLink(fileLink)); - // FIXME - store->addTempRoot(*storePath); - if (store->isValidPath(*storePath)) { - auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath.reset(); - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path.clone(); - } - - assert(storePath); - replaceSymlink(store->printStorePath(*storePath), fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (!storePath) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; - } - } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - std::optional<StorePath> unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); - // FIXME - store->addTempRoot(*unpackedStorePath); - if (!store->isValidPath(*unpackedStorePath)) - unpackedStorePath.reset(); - } - if (!unpackedStorePath) { - printInfo("unpacking '%s'...", url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink); - storePath = std::move(*unpackedStorePath); - } - - if (expectedStorePath && *storePath != *expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first - : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); - } - - result.storePath = store->printStorePath(*storePath); - result.path = store->toRealPath(result.storePath); - return result; -} - - bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/download.hh b/src/libstore/filetransfer.hh index 5a131c704..11dca2fe0 100644 --- a/src/libstore/download.hh +++ b/src/libstore/filetransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DownloadSettings : Config +struct FileTransferSettings : Config { Setting<bool> enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,15 +31,15 @@ struct DownloadSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DownloadSettings downloadSettings; +extern FileTransferSettings fileTransferSettings; -struct DownloadRequest +struct FileTransferRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = downloadSettings.tries; + size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -47,7 +47,7 @@ struct DownloadRequest std::string mimeType; std::function<void(char *, size_t)> dataCallback; - DownloadRequest(const std::string & uri) + FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -56,7 +56,7 @@ struct DownloadRequest } }; -struct DownloadResult +struct FileTransferResult { bool cached = false; std::string etag; @@ -65,74 +65,53 @@ struct DownloadResult uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl; - - CachedDownloadRequest(const std::string & uri); - CachedDownloadRequest() = delete; -}; - -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional<std::string> etag; - std::string effectiveUri; -}; - class Store; -struct Downloader +struct FileTransfer { - virtual ~Downloader() { } + virtual ~FileTransfer() { } - /* Enqueue a download request, returning a future to the result of - the download. The future may throw a DownloadError + /* Enqueue a data transfer request, returning a future to the result of + the download. The future may throw a FileTransferError exception. */ - virtual void enqueueDownload(const DownloadRequest & request, - Callback<DownloadResult> callback) = 0; + virtual void enqueueFileTransfer(const FileTransferRequest & request, + Callback<FileTransferResult> callback) = 0; - std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); + std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); + FileTransferResult download(const FileTransferRequest & request); + + /* Synchronously upload a file. */ + FileTransferResult upload(const FileTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DownloadRequest && request, Sink & sink); - - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); + void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared Downloader object. Using this object is preferred +/* Return a shared FileTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref<Downloader> getDownloader(); +ref<FileTransfer> getFileTransfer(); -/* Return a new Downloader object. */ -ref<Downloader> makeDownloader(); +/* Return a new FileTransfer object. */ +ref<FileTransfer> makeFileTransfer(); -class DownloadError : public Error +class FileTransferError : public Error { public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) - : Error(fs), error(error) + FileTransfer::Error error; + template<typename... Args> + FileTransferError(FileTransfer::Error error, const Args & ... args) + : Error(args...), error(error) { } }; bool isUri(const string & s); +/* Resolve deprecated 'channel:<foo>' URLs. */ +std::string resolveUri(const std::string & uri); + } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0c3d89611..57fb20845 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -38,10 +38,10 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fdGCLock) - throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - printError(format("waiting for the big garbage collector lock...")); + printInfo("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } @@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target) /* Atomically replace the old one. */ if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename '%1%' to '%2%'") - % tempLink % link); + throw SysError("cannot rename '%1%' to '%2%'", + tempLink , link); } @@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path gcRoot(canonPath(_gcRoot)); if (isInStore(gcRoot)) - throw Error(format( + throw Error( "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); + "(are you running nix-build inside the store?)", gcRoot); if (indirect) { /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); + throw Error("cannot create symlink '%1%'; already exists", gcRoot); makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( + throw Error( "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'") - % gcRoot % rootsDir); + "it's not in the directory '%2%'", + gcRoot, rootsDir); } if (baseNameOf(gcRoot) == std::string(storePath.to_string())) @@ -128,11 +128,13 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, gcroots directory. */ if (settings.checkRootReachability) { auto roots = findRoots(false); - if (roots[storePath.clone()].count(gcRoot) == 0) - printError( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)); + if (roots[storePath].count(gcRoot) == 0) + logWarning({ + .name = "GC root", + .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)) + }); } /* Grab the global GC root, causing us to block while a GC is in @@ -170,7 +172,7 @@ void LocalStore::addTempRoot(const StorePath & path) way. */ struct stat st; if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError(format("statting '%1%'") % fnTempRoots); + throw SysError("statting '%1%'", fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -202,6 +204,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { + if (i.name[0] == '.') { + // Ignore hidden files. Some package managers (notably portage) create + // those to keep the directory alive. + continue; + } Path path = tempRootsDir + "/" + i.name; pid_t pid = std::stoi(i.name); @@ -211,7 +218,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) if (!*fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file '%1%'") % path); + throw SysError("opening temporary roots file '%1%'", path); } /* This should work, but doesn't, for some reason. */ @@ -222,7 +229,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); + printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); writeFull(fd->get(), "d"); continue; @@ -398,7 +405,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!fdDir) { if (errno == ENOENT || errno == EACCES) continue; - throw SysError(format("opening %1%") % fdStr); + throw SysError("opening %1%", fdStr); } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { @@ -408,13 +415,13 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (errno) { if (errno == ESRCH) continue; - throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + throw SysError("iterating /proc/%1%/fd", ent->d_name); } fdDir.reset(); try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); + auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) @@ -422,7 +429,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile, true); + auto envString = readFile(envFile); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); @@ -471,9 +478,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!isValidPath(path)) continue; debug("got additional root '%1%'", pathS); if (censor) - roots[path.clone()].insert(censored); + roots[path].insert(censored); else - roots[path.clone()].insert(links.begin(), links.end()); + roots[path].insert(links.begin(), links.end()); } } @@ -536,7 +543,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % realPath); + throw SysError("getting status of %1%", realPath); } printInfo(format("deleting '%1%'") % path); @@ -554,10 +561,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) // size. try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making '%1%' writable") % realPath); + throw SysError("making '%1%' writable", realPath); Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) - throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); + throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); state.bytesInvalidated += size; } catch (SysError & e) { if (e.errNo == ENOSPC) { @@ -585,11 +592,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.roots.count(path)) { debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } - visited.insert(path.clone()); + visited.insert(path); if (!isValidPath(path)) return false; @@ -603,7 +610,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepDerivations && path.isDerivation()) { for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i.clone()); + incoming.insert(i); } /* If keep-outputs is set, then don't delete this path if there @@ -611,13 +618,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepOutputs) { auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i.clone()); + incoming.insert(i); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } @@ -661,7 +668,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ for (auto & i : visited) - state.dead.insert(i.clone()); + state.dead.insert(i); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -676,7 +683,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) void LocalStore::removeUnusedLinks(const GCState & state) { AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); long long actualSize = 0, unsharedSize = 0; @@ -689,7 +696,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % path); + throw SysError("statting '%1%'", path); if (st.st_nlink != 1) { actualSize += st.st_size; @@ -700,14 +707,14 @@ void LocalStore::removeUnusedLinks(const GCState & state) printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); if (unlink(path.c_str()) == -1) - throw SysError(format("deleting '%1%'") % path); + throw SysError("deleting '%1%'", path); state.results.bytesFreed += st.st_size; } struct stat st; if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % linksDir); + throw SysError("statting '%1%'", linksDir); long long overhead = st.st_blocks * 512ULL; printInfo(format("note: currently hard linking saves %.2f MiB") @@ -742,12 +749,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError("finding garbage collector roots..."); + printInfo("finding garbage collector roots..."); Roots rootMap; if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first.clone()); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -756,8 +763,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Roots tempRoots; findTempRoots(fds, tempRoots, true); for (auto & root : tempRoots) { - state.tempRoots.insert(root.first.clone()); - state.roots.insert(root.first.clone()); + state.tempRoots.insert(root.first); + state.roots.insert(root.first); } /* After this point the set of roots or temporary roots cannot @@ -794,14 +801,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError("deleting garbage..."); + printInfo("deleting garbage..."); else - printError("determining live/dead paths..."); + printInfo("determining live/dead paths..."); try { AutoCloseDir dir(opendir(realStoreDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); + if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that aren't valid. When using --max-freed etc., deleting @@ -863,7 +870,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError("deleting unused links..."); + printInfo("deleting unused links..."); removeUnusedLinks(state); } @@ -884,7 +891,7 @@ void LocalStore::autoGC(bool sync) if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); - return (uint64_t) st.f_bavail * st.f_bsize; + return (uint64_t) st.f_bavail * st.f_frsize; }; std::shared_future<void> future; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index a8945996e..bee94cbd8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -20,13 +20,6 @@ namespace nix { must be deleted and recreated on startup.) */ #define DEFAULT_SOCKET_PATH "/daemon-socket/socket" -/* chroot-like behavior from Apple's sandbox */ -#if __APPLE__ - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" -#else - #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" -#endif - Settings settings; static GlobalConfig::Register r1(&settings); @@ -38,6 +31,7 @@ Settings::Settings() , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixUserConfFiles(getUserConfigFiles()) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) @@ -68,7 +62,12 @@ Settings::Settings() sandboxPaths = tokenizeString<StringSet>("/bin/sh=" SANDBOX_SHELL); #endif - allowedImpureHostPrefixes = tokenizeString<StringSet>(DEFAULT_ALLOWED_IMPURE_PREFIXES); + +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + sandboxPaths = tokenizeString<StringSet>("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); + allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh"); +#endif } void loadConfFile() @@ -79,11 +78,27 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); + auto files = settings.nixUserConfFiles; + for (auto file = files.rbegin(); file != files.rend(); file++) { + globalConfig.applyConfigFile(*file); + } +} + +std::vector<Path> getUserConfigFiles() +{ + // Use the paths specified in NIX_USER_CONF_FILES if it has been defined + auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); + if (nixConfFiles.has_value()) { + return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":"); + } + + // Use the paths specified by the XDG spec + std::vector<Path> files; auto dirs = getConfigDirs(); - // Iterate over them in reverse so that the ones appearing first in the path take priority - for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { - globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + for (auto & dir : dirs) { + files.insert(files.end(), dir + "/nix/nix.conf"); } + return files; } unsigned int Settings::getDefaultCores() @@ -115,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%s' is disabled", name); + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); } bool Settings::isWSL1() @@ -152,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description("Enable sandboxing.") - .handler([=](std::vector<std::string> ss) { override(smEnabled); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description("Disable sandboxing.") - .handler([=](std::vector<std::string> ss) { override(smDisabled); }) - .category(category); - args.mkFlag() - .longName("relaxed-" + name) - .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](std::vector<std::string> ss) { override(smRelaxed); }) - .category(category); + args.addFlag({ + .longName = name, + .description = "Enable sandboxing.", + .category = category, + .handler = {[=]() { override(smEnabled); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = "Disable sandboxing.", + .category = category, + .handler = {[=]() { override(smDisabled); }} + }); + args.addFlag({ + .longName = "relaxed-" + name, + .description = "Enable sandboxing, but allow builds to disable it.", + .category = category, + .handler = {[=]() { override(smRelaxed); }} + }); } void MaxBuildJobsSetting::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 782870547..2fbcafff8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -53,9 +53,12 @@ public: /* The directory where state is stored. */ Path nixStateDir; - /* The directory where configuration files are stored. */ + /* The directory where system configuration files are stored. */ Path nixConfDir; + /* A list of user configuration files to load. */ + std::vector<Path> nixUserConfFiles; + /* The directory where internal helper programs are stored. */ Path nixLibexecDir; @@ -268,7 +271,7 @@ public: "listed in 'trusted-public-keys'."}; Setting<StringSet> extraPlatforms{this, - std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", "Additional platforms that can be built on the local system. " "These may be supported natively (e.g. armv7 on some aarch64 CPUs " @@ -311,12 +314,7 @@ public: Setting<bool> printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; - Setting<std::string> preBuildHook{this, -#if __APPLE__ - nixLibexecDir + "/nix/resolve-system-dependencies", -#else - "", -#endif + Setting<std::string> preBuildHook{this, "", "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; @@ -356,12 +354,21 @@ public: Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + Setting<std::string> githubAccessToken{this, "", "github-access-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting<Strings> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; bool isExperimentalFeatureEnabled(const std::string & name); void requireExperimentalFeature(const std::string & name); + + Setting<bool> allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting<bool> warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; @@ -374,6 +381,9 @@ void initPlugins(); void loadConfFile(); +// Used by the Settings constructor +std::vector<Path> getUserConfigFiles(); + extern const string nixVersion; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index d4ae36662..451a64785 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -85,14 +85,14 @@ protected: checkEnabled(); try { - DownloadRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); request.head = true; - getDownloader()->download(request); + getFileTransfer()->download(request); return true; - } catch (DownloadError & e) { + } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; maybeDisable(); throw; @@ -103,19 +103,19 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DownloadRequest(cacheUri + "/" + path); + auto req = FileTransferRequest(cacheUri + "/" + path); req.data = std::make_shared<string>(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDownloader()->download(req); - } catch (DownloadError & e) { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } - DownloadRequest makeRequest(const std::string & path) + FileTransferRequest makeRequest(const std::string & path) { - DownloadRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); return request; } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDownloader()->download(std::move(request), sink); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - getDownloader()->enqueueDownload(request, - {[callbackPtr, this](std::future<DownloadResult> result) { + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future<FileTransferResult> result) { try { (*callbackPtr)(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr<std::string>()); maybeDisable(); callbackPtr->rethrow(); @@ -163,14 +163,14 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://" && - (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") - ) return 0; + (!forceHttp || std::string(uri, 0, 7) != "file://")) + return 0; auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); store->init(); return store; }); } - diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 458266be0..45c70fad6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -195,7 +195,7 @@ struct LegacySSHStore : public Store { unsupported("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override { unsupported("addToStore"); } @@ -256,7 +256,7 @@ struct LegacySSHStore : public Store conn->to.flush(); for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) - out.insert(i.clone()); + out.insert(i); } StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 363be1443..48aca478c 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s) AutoDelete del(tmp, false); writeFile(tmp, s); if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + throw SysError("renaming '%1%' to '%2%'", tmp, path); del.cancel(); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index aa5abd835..dd96d2578 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor { Path storePath = store->toStorePath(path); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error(format("file '%1%' has unsupported type") % path); + throw Error("file '%1%' has unsupported type", path); return { S_ISREG(st.st_mode) ? Type::tRegular : @@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs"; std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) { - auto path = path_.clone(); + auto path = path_; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); if (!info->deriver) return nullptr; - path = info->deriver->clone(); + path = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8639cbf20..460be97a1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -87,18 +87,22 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + logError({ + .name = "'build-users-group' not found", + .hint = hintfmt( + "warning: the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup) + }); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % realStoreDir); + throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path '%1%'") % realStoreDir); + throw SysError("changing ownership of path '%1%'", realStoreDir); if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError(format("changing permissions on path '%1%'") % realStoreDir); + throw SysError("changing permissions on path '%1%'", realStoreDir); } } } @@ -109,12 +113,12 @@ LocalStore::LocalStore(const Params & params) struct stat st; while (path != "/") { if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); if (S_ISLNK(st.st_mode)) - throw Error(format( + throw Error( "the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); + "this is not allowed for the Nix store and its parent directories", + path); path = dirOf(path); } } @@ -147,7 +151,7 @@ LocalStore::LocalStore(const Params & params) globalLock = openLockFile(globalLockPath.c_str(), true); if (!lockFile(globalLock.get(), ltRead, false)) { - printError("waiting for the big Nix store lock..."); + printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -155,8 +159,8 @@ LocalStore::LocalStore(const Params & params) upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); + throw Error("current Nix store schema is version %1%, but I only support %2%", + curSchema, nixSchemaVersion); else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; @@ -178,7 +182,7 @@ LocalStore::LocalStore(const Params & params) "please upgrade Nix to version 1.11 first."); if (!lockFile(globalLock.get(), ltWrite, false)) { - printError("waiting for exclusive access to the Nix store..."); + printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltWrite, true); } @@ -256,7 +260,7 @@ LocalStore::~LocalStore() } if (future.valid()) { - printError("waiting for auto-GC to finish on exit..."); + printInfo("waiting for auto-GC to finish on exit..."); future.get(); } @@ -284,7 +288,7 @@ int LocalStore::getSchema() if (pathExists(schemaPath)) { string s = readFile(schemaPath); if (!string2Int(s, curSchema)) - throw Error(format("'%1%' is corrupt") % schemaPath); + throw Error("'%1%' is corrupt", schemaPath); } return curSchema; } @@ -293,14 +297,12 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory '%1%' is not writable") % dbDir); + throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - if (sqlite3_open_v2(dbPath.c_str(), &db.db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open Nix database '%1%'") % dbPath); + state.db = SQLite(dbPath, create); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls @@ -312,11 +314,6 @@ void LocalStore::openDB(State & state, bool create) SetDllDirectoryW(L""); #endif - if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); - - db.exec("pragma foreign_keys = 1"); - /* !!! check whether sqlite has been built with foreign key support */ @@ -350,7 +347,7 @@ void LocalStore::openDB(State & state, bool create) /* Initialise the database schema, if necessary. */ if (create) { - const char * schema = + static const char schema[] = #include "schema.sql.gen.hh" ; db.exec(schema); @@ -374,7 +371,7 @@ void LocalStore::makeStoreWritable() throw SysError("setting up a private mount namespace"); if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % realStoreDir); + throw SysError("remounting %1% writable", realStoreDir); } #endif } @@ -395,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct | 0444 | (st.st_mode & S_IXUSR ? 0111 : 0); if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + throw SysError("changing mode of '%1%' to %2$o", path, mode); } } @@ -413,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #else if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) #endif - throw SysError(format("changing modification time of '%1%'") % path); + throw SysError("changing modification time of '%1%'", path); } } @@ -422,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); canonicaliseTimestampAndPermissions(path, st); } @@ -437,17 +434,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe setattrlist() to remove other attributes as well. */ if (lchflags(path.c_str(), 0)) { if (errno != ENOTSUP) - throw SysError(format("clearing flags of path '%1%'") % path); + throw SysError("clearing flags of path '%1%'", path); } #endif struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error(format("file '%1%' has an unsupported type") % path); + throw Error("file '%1%' has an unsupported type", path); #if __linux__ /* Remove extended attributes / ACLs. */ @@ -481,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file '%1%'") % path); + throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); return; @@ -505,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) #endif - throw SysError(format("changing owner of '%1%' to %2%") - % path % geteuid()); + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); } if (S_ISDIR(st.st_mode)) { @@ -525,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino be a symlink, since we can't change its ownership. */ struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path '%1%'") % path); + throw Error("wrong ownership of top-level store path '%1%'", path); } } @@ -564,10 +561,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); } else { @@ -589,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -604,7 +601,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); + auto drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -624,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), + state_->pathInfoCache.upsert(std::string(info.path.hashPart()), PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) }); } @@ -636,7 +633,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(path.clone()); + auto info = std::make_shared<ValidPathInfo>(path); callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); @@ -689,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -726,7 +723,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } @@ -794,26 +791,9 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) } -StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) -{ - return retrySQLite<StringSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - - return outputNames; - }); -} - - std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) { - if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); + if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; @@ -838,7 +818,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining; for (auto & i : paths) - remaining.insert(i.clone()); + remaining.insert(i); StorePathSet res; @@ -852,9 +832,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path.clone()); + res.insert(path); else - remaining2.insert(path.clone()); + remaining2.insert(path); std::swap(remaining, remaining2); } @@ -876,16 +856,16 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ - info->deriver ? info->deriver->clone() : std::optional<StorePath>(), - cloneStorePathSet(info->references), + infos.insert_or_assign(path, SubstitutablePathInfo{ + info->deriver, + info->references, narInfo ? narInfo->fileSize : 0, info->narSize}); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) - printError(e.what()); + logError(e.info()); else throw; } @@ -922,7 +902,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path.clone()); + paths.insert(i.path); } for (auto & i : infos) { @@ -937,8 +917,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); + checkDerivationOutputs(i.path, readDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an @@ -965,7 +944,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); + state_->pathInfoCache.erase(std::string(path.hashPart())); } } @@ -1017,7 +996,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique<HashSink>(htSHA256); else - hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path))); + hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); @@ -1031,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1052,11 +1031,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -1076,7 +1055,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (recursive) { + if (method == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -1089,7 +1068,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1097,10 +1076,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); + info.ca = makeFixedOutputCA(method, h); registerValidPath(info); } @@ -1112,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1120,12 +1099,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref<std::string>(readFile(srcPath)); - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } @@ -1160,11 +1139,11 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = cloneStorePathSet(references); - info.ca = "text:" + hash.to_string(); + info.references = references; + info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } @@ -1214,7 +1193,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printError(format("reading the Nix store...")); + printInfo(format("reading the Nix store...")); bool errors = false; @@ -1246,12 +1225,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - printError( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash); + logError({ + .name = "Invalid hash", + .hint = hintfmt( + "link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash) + }); if (repair) { if (unlink(linkPath.c_str()) == 0) - printError("removed link '%s'", linkPath); + printInfo("removed link '%s'", linkPath); else throw SysError("removing corrupt link '%s'", linkPath); } else { @@ -1275,14 +1257,17 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->ca == "" || !info->references.count(info->path)) hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path))); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); - dumpPath(toRealPath(printStorePath(i)), *hashSink); + dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + logError({ + .name = "Invalid hash - path modified", + .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + }); if (repair) repairPath(i); else errors = true; } else { @@ -1290,14 +1275,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError("fixing missing hash on '%s'", printStorePath(i)); + printInfo("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError("updating size field on '%s' to %s", printStorePath(i), current.second); + printInfo("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1313,7 +1298,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError("error: %s", e.msg()); + logError(e.info()); else warn(e.msg()); errors = true; @@ -1333,7 +1318,10 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); + logError({ + .name = "Nix path not found", + .hint = hintfmt("path '%s' is not in the Nix store", pathS) + }); return; } @@ -1352,16 +1340,19 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, } if (canInvalidate) { - printError("path '%s' disappeared, removing from database...", pathS); + printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { - printError("path '%s' disappeared, but it still has valid referrers!", pathS); + logError({ + .name = "Missing path with referrers", + .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) + }); if (repair) try { repairPath(path); } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); errors = true; } else errors = true; @@ -1401,7 +1392,7 @@ static void makeMutable(const Path & path) AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) { if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); } unsigned int flags = 0, old; @@ -1419,7 +1410,7 @@ static void makeMutable(const Path & path) void LocalStore::upgradeStore7() { if (getuid() != 0) return; - printError("removing immutable bits from the Nix store (this may take a while)..."); + printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 16aeab0ad..e17cc45ae 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -135,8 +135,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -149,7 +147,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; /* Like addToStore(), but the contents of the path are contained @@ -157,7 +155,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ac68c2342..aec4ed493 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) @@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif -libstore_CXXFLAGS = \ +libstore_CXXFLAGS += \ + -I src/libutil -I src/libstore \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..e68edb38c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -103,7 +103,7 @@ void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { StorePathSet paths; - paths.insert(startPath.clone()); + paths.insert(startPath); computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } @@ -141,11 +141,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath.clone()); + state->willBuild.insert(drvPath); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); }; auto checkOutput = [&]( @@ -157,9 +157,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({outPath}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -170,10 +168,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath.clone()); + drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); } } } @@ -190,12 +188,12 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } auto drv = make_ref<Derivation>(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(path.path.clone(), *drv); + ParsedDerivation parsedDrv(StorePath(path.path), *drv); PathSet invalid; for (auto & j : drv->outputs) @@ -216,13 +214,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(path.path.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({path.path}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } @@ -231,13 +227,13 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, { auto state(state_.lock()); - state->willSubstitute.insert(path.path.clone()); + state->willSubstitute.insert(path.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); } }; @@ -260,12 +256,12 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) throw BuildError("cycle detected in the references of '%s' from '%s'", printStorePath(path), printStorePath(*parent)); - if (!visited.insert(path.clone()).second) return; - parents.insert(path.clone()); + if (!visited.insert(path).second) return; + parents.insert(path); StorePathSet references; try { - references = cloneStorePathSet(queryPathInfo(path)->references); + references = queryPathInfo(path)->references; } catch (InvalidPath &) { } @@ -275,7 +271,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_back(path.clone()); + sorted.push_back(path); parents.erase(path); }; diff --git a/src/libexpr/names.cc b/src/libstore/names.cc index d1c8a6101..d1c8a6101 100644 --- a/src/libexpr/names.cc +++ b/src/libstore/names.cc diff --git a/src/libexpr/names.hh b/src/libstore/names.hh index 00e14b8c7..00e14b8c7 100644 --- a/src/libexpr/names.hh +++ b/src/libstore/names.hh diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b74480684..ca663d837 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor auto i = get(path); if (i.type != FSAccessor::Type::tDirectory) - throw Error(format("path '%1%' inside NAR file is not a directory") % path); + throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; for (auto & child : i.children) @@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' inside NAR file is not a regular file") % path); + throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.start, i.size); @@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tSymlink) - throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 907645d86..552970248 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -78,12 +78,7 @@ public: state->db = SQLite(dbPath); - if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(state->db, "setting timeout"); - - // We can always reproduce the cache. - state->db.exec("pragma synchronous = off"); - state->db.exec("pragma main.journal_mode = truncate"); + state->db.isCache(); state->db.exec(schema); @@ -194,7 +189,7 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart)); + auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart)); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) @@ -203,9 +198,9 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath::fromBaseName(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) - narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); + narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); narInfo->ca = queryNAR.getStr(11); @@ -235,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 878acbb87..04de2c5eb 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -10,7 +10,7 @@ class NarInfoDiskCache public: typedef enum { oValid, oInvalid, oUnknown } Outcome; - virtual ~NarInfoDiskCache() { }; + virtual ~NarInfoDiskCache() { } virtual void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..bb4448c90 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,10 +4,10 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error(format("NAR info file '%1%' is corrupt") % whence); + throw Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { @@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath::fromBaseName(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") - deriver = StorePath::fromBaseName(value); + deriver = StorePath(value); } else if (name == "System") system = value; @@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..b2b2412a3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -19,9 +19,9 @@ static void makeWritable(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of '%1%'") % path); + throw SysError("changing writability of '%1%'", path); } @@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() InodeHash inodeHash; AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + if (errno) throw SysError("reading directory '%1%'", linksDir); printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); @@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa Strings names; AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + if (!dir) throw SysError("opening directory '%1%'", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa if (name == "." || name == "..") continue; names.push_back(name); } - if (errno) throw SysError(format("reading directory '%1%'") % path); + if (errno) throw SysError("reading directory '%1%'", path); return names; } @@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -130,7 +130,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file '%1%'") % path); + logWarning({ + .name = "Suspicious file", + .hint = hintfmt("skipping suspicious writable file '%1%'", path) + }); return; } @@ -150,7 +153,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); @@ -186,7 +189,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, current file with a hard link to that file. */ struct stat stLink; if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path '%1%'") % linkPath); + throw SysError("getting attributes of path '%1%'", linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); @@ -194,7 +197,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link '%1%'") % linkPath); + logWarning({ + .name = "Corrupted link", + .hint = hintfmt("removing corrupted link '%1%'", linkPath) + }); unlink(linkPath.c_str()); goto retry; } @@ -229,7 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink '%1%'") % tempLink); + logError({ + .name = "Unlink error", + .hint = hintfmt("unable to unlink '%1%'", tempLink) + }); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it @@ -238,7 +247,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); } stats.filesLinked++; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index d0f289a0f..24f848e46 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,21 +1,25 @@ #include "parsed-derivations.hh" +#include <nlohmann/json.hpp> + namespace nix { -ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) - : drvPath(std::move(drvPath)), drv(drv) +ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); if (jsonAttr != drv.env.end()) { try { - structuredAttrs = nlohmann::json::parse(jsonAttr->second); + structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); } } } +ParsedDerivation::~ParsedDerivation() { } + std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const { if (structuredAttrs) { diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index cec868754..7621342d7 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,6 +1,6 @@ #include "derivations.hh" -#include <nlohmann/json.hpp> +#include <nlohmann/json_fwd.hpp> namespace nix { @@ -8,15 +8,17 @@ class ParsedDerivation { StorePath drvPath; BasicDerivation & drv; - std::optional<nlohmann::json> structuredAttrs; + std::unique_ptr<nlohmann::json> structuredAttrs; public: - ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); + ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv); - const std::optional<nlohmann::json> & getStructuredAttrs() const + ~ParsedDerivation(); + + const nlohmann::json * getStructuredAttrs() const { - return structuredAttrs; + return structuredAttrs.get(); } std::optional<std::string> getStringAttr(const std::string & name) const; diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 9a28aa96a..b3d8ce95c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,38 +2,38 @@ namespace nix { -extern "C" { - rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); - rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); - rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName); - rust::String ffi_StorePath_to_string(const StorePath & _this); - StorePath ffi_StorePath_clone(const StorePath & _this); - rust::StringSlice ffi_StorePath_name(const StorePath & _this); -} - -StorePath StorePath::make(std::string_view path, std::string_view storeDir) -{ - return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); -} - -StorePath StorePath::make(unsigned char hash[20], std::string_view name) -{ - return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); -} +MakeError(BadStorePath, Error); -StorePath StorePath::fromBaseName(std::string_view baseName) +static void checkName(std::string_view path, std::string_view name) { - return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); + if (name.empty()) + throw BadStorePath("store path '%s' has an empty name", path); + if (name.size() > 211) + throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + for (auto c : name) + if (!((c >= '0' && c <= '9') + || (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) + throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); } -rust::String StorePath::to_string() const +StorePath::StorePath(std::string_view _baseName) + : baseName(_baseName) { - return ffi_StorePath_to_string(*this); + if (baseName.size() < HashLen + 1) + throw BadStorePath("'%s' is too short to be a valid store path", baseName); + for (auto c : hashPart()) + if (c == 'e' || c == 'o' || c == 'u' || c == 't' + || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) + throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); + checkName(baseName, name()); } -StorePath StorePath::clone() const +StorePath::StorePath(const Hash & hash, std::string_view _name) + : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) { - return ffi_StorePath_clone(*this); + checkName(baseName, name()); } bool StorePath::isDerivation() const @@ -41,18 +41,14 @@ bool StorePath::isDerivation() const return hasSuffix(name(), drvExtension); } -std::string_view StorePath::name() const -{ - return ffi_StorePath_name(*this); -} - -StorePath StorePath::dummy( - StorePath::make( - (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")); +StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath Store::parseStorePath(std::string_view path) const { - return StorePath::make(path, storeDir); + auto p = canonPath(std::string(path)); + if (dirOf(p) != storeDir) + throw BadStorePath("path '%s' is not in the Nix store", p); + return StorePath(baseNameOf(p)); } std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const @@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const std::string Store::printStorePath(const StorePath & path) const { - auto s = storeDir + "/"; - s += (std::string_view) path.to_string(); - return s; + return (storeDir + "/").append(path.to_string()); } PathSet Store::printStorePathSet(const StorePathSet & paths) const @@ -90,29 +84,6 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -StorePathSet cloneStorePathSet(const StorePathSet & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet storePathsToSet(const StorePaths & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet singleton(const StorePath & path) -{ - StorePathSet res; - res.insert(path.clone()); - return res; -} - std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index c90bb1fff..aaebd3ec3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,59 +1,59 @@ #pragma once -#include "rust-ffi.hh" +#include "types.hh" namespace nix { -/* See path.rs. */ -struct StorePath; - class Store; +struct Hash; -extern "C" { - void ffi_StorePath_drop(void *); - bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); - bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); - unsigned char * ffi_StorePath_hash_data(const StorePath & p); -} - -struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +class StorePath { - StorePath() = delete; + std::string baseName; + +public: - static StorePath make(std::string_view path, std::string_view storeDir); + /* Size of the hash part of store paths, in base-32 characters. */ + constexpr static size_t HashLen = 32; // i.e. 160 bits - static StorePath make(unsigned char hash[20], std::string_view name); + StorePath() = delete; - static StorePath fromBaseName(std::string_view baseName); + StorePath(std::string_view baseName); - rust::String to_string() const; + StorePath(const Hash & hash, std::string_view name); + + std::string_view to_string() const + { + return baseName; + } bool operator < (const StorePath & other) const { - return ffi_StorePath_less_than(*this, other); + return baseName < other.baseName; } bool operator == (const StorePath & other) const { - return ffi_StorePath_eq(*this, other); + return baseName == other.baseName; } bool operator != (const StorePath & other) const { - return !(*this == other); + return baseName != other.baseName; } - StorePath clone() const; - /* Check whether a file name ends with the extension for derivations. */ bool isDerivation() const; - std::string_view name() const; + std::string_view name() const + { + return std::string_view(baseName).substr(HashLen + 1); + } - unsigned char * hashData() const + std::string_view hashPart() const { - return ffi_StorePath_hash_data(*this); + return std::string_view(baseName).substr(0, HashLen); } static StorePath dummy; @@ -62,34 +62,19 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -StorePathSet cloneStorePathSet(const StorePathSet & paths); -StorePathSet storePathsToSet(const StorePaths & paths); - -StorePathSet singleton(const StorePath & path); - -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + struct StorePathWithOutputs { StorePath path; std::set<std::string> outputs; - StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {}) - : path(path.clone()), outputs(outputs) - { } - - StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs) - : path(std::move(path)), outputs(std::move(outputs)) - { } - - StorePathWithOutputs(const StorePathWithOutputs & other) - : path(other.path.clone()), outputs(other.outputs) - { } - std::string to_string(const Store & store) const; }; @@ -102,7 +87,7 @@ namespace std { template<> struct hash<nix::StorePath> { std::size_t operator()(const nix::StorePath & path) const noexcept { - return * (std::size_t *) path.hashData(); + return * (std::size_t *) path.to_string().data(); } }; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 2635e3940..926f4ea1e 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create) fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); if (!fd && (create || errno != ENOENT)) - throw SysError(format("opening lock file '%1%'") % path); + throw SysError("opening lock file '%1%'", path); return fd; } @@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait) while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); else return false; } @@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait) checkInterrupt(); if (errno == EWOULDBLOCK) return false; if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); } } @@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths, hasn't been unlinked). */ struct stat st; if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); + throw SysError("statting lock file '%1%'", lockPath); if (st.st_size != 0) /* This lock file has been unlinked, so we're holding a lock on a deleted file. This means that other @@ -160,7 +160,8 @@ void PathLocks::unlock() if (close(i.first) == -1) printError( - format("error (ignored): cannot close lock file on '%1%'") % i.second); + "error (ignored): cannot close lock file on '%1%'", + i.second); debug(format("lock released on '%1%'") % i.second); } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index dae3f2d32..6cfe393a4 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen) gen.number = n; struct stat st; if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting '%1%'") % gen.path); + throw SysError("statting '%1%'", gen.path); gen.creationTime = st.st_mtime; gens.push_back(gen); } @@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) static void removeFile(const Path & path) { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } @@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens Generations gens = findGenerations(profile, curGen); if (gensToDelete.find(curGen) != gensToDelete.end()) - throw Error(format("cannot delete current generation of profile %1%'") % profile); + throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { if (gensToDelete.find(i.number) == gensToDelete.end()) continue; @@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b int days; if (!string2Int(strDays, days) || days < 1) - throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + throw Error("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - days * 24 * 3600; @@ -256,4 +256,22 @@ string optimisticLockProfile(const Path & profile) } +Path getDefaultProfile() +{ + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + return absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + return profileLink; + } +} + + } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 5fa1533de..78645d8b6 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -64,4 +64,8 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ string optimisticLockProfile(const Path & profile); +/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create + it. */ +Path getDefaultProfile(); + } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 102e15921..a10d536a3 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -92,7 +92,7 @@ PathSet scanForReferences(const string & path, auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) - throw Error(format("bad reference '%1%'") % i); + throw Error("bad reference '%1%'", i); string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5a2d103b9..bd698d781 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); + return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); } void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, @@ -51,7 +51,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) std::string restPath = std::string(path, storePath.size()); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); auto i = nars.find(storePath); if (i != nars.end()) return {i->second, restPath}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..fc5ab5865 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -116,11 +116,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() struct sockaddr_un addr; addr.sun_family = AF_UNIX; if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPath); + throw Error("socket path '%1%' is too long", socketPath); strcpy(addr.sun_path, socketPath.c_str()); if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); + throw SysError("cannot connect to daemon at '%1%'", socketPath); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -268,7 +268,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } else { conn->to << wopQueryValidPaths; @@ -296,7 +296,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & i : paths) { conn->to << wopHasSubstitutes << printStorePath(i); conn.processStderr(); - if (readInt(conn->from)) res.insert(i.clone()); + if (readInt(conn->from)) res.insert(i); } return res; } else { @@ -329,7 +329,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.clone(), std::move(info)); + infos.insert_or_assign(i, std::move(info)); } } else { @@ -365,14 +365,14 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); + throw InvalidPath(e.info()); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(path.clone()); + info = std::make_shared<ValidPathInfo>(StorePath(path)); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); @@ -396,7 +396,7 @@ void RemoteStore::queryReferrers(const StorePath & path, conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) - referrers.insert(i.clone()); + referrers.insert(i); } @@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << printStorePath(path); - conn.processStderr(); - return readStrings<PathSet>(conn->from); -} - - std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); @@ -484,7 +475,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -492,10 +483,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, Path srcPath(absPath(_srcPath)); - conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ - << (recursive ? 1 : 0) - << printHashType(hashAlgo); + conn->to + << wopAddToStore + << name + << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashAlgo); try { conn->to.written = 0; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f301a97d8..80c8e9f11 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,8 +51,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -65,7 +63,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f2e4b63e0..427dd48ce 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "download.hh" +#include "filetransfer.hh" #include "istringstream_nocopy.hh" #include <aws/core/Aws.h> @@ -32,8 +32,10 @@ namespace nix { struct S3Error : public Error { Aws::S3::S3Errors err; - S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) - : Error(fs), err(err) { }; + + template<typename... Args> + S3Error(Aws::S3::S3Errors err, const Args & ... args) + : Error(args...), err(err) { }; }; /* Helper: given an Outcome<R, E>, return R in case of success, or @@ -109,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) printError("AWS error '%s' (%s), will retry in %d ms", - error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); + error.GetExceptionName(), + error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } }; @@ -132,7 +136,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region return res; } -S3Helper::DownloadResult S3Helper::getObject( +S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +150,7 @@ S3Helper::DownloadResult S3Helper::getObject( return Aws::New<std::stringstream>("STRINGSTREAM"); }); - DownloadResult res; + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); @@ -249,7 +253,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore // If bucket listing is disabled, 404s turn into 403s || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) return false; - throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); + throw Error("AWS error fetching '%s': %s", path, error.GetMessage()); } return true; diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index ef5f23d0f..2042bffcf 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DownloadResult + struct FileTransferResult { std::shared_ptr<std::string> data; unsigned int durationMs; }; - DownloadResult getObject( + FileTransferResult getObject( const std::string & bucketName, const std::string & key); }; diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 0299d1ee4..351037822 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -71,6 +71,12 @@ (literal "/dev/zero") (subpath "/dev/fd")) +; Allow pseudo-terminals. +(allow file* + (literal "/dev/ptmx") + (regex #"^/dev/pty[a-z]+") + (regex #"^/dev/ttys[0-9]+")) + ; Does nothing, but reduces build noise. (allow file* (literal "/dev/dtracehelper")) @@ -85,3 +91,7 @@ (literal "/etc") (literal "/var") (literal "/private/var/tmp")) + +; This is used by /bin/sh on macOS 10.15 and later. +(allow file* + (literal "/private/var/select/sh")) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index a061d64f3..76c822c4e 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -25,11 +25,16 @@ namespace nix { throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); } -SQLite::SQLite(const Path & path) +SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error(format("cannot open SQLite database '%s'") % path); + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error("cannot open SQLite database '%s'", path); + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + exec("pragma foreign_keys = 1"); } SQLite::~SQLite() @@ -42,6 +47,12 @@ SQLite::~SQLite() } } +void SQLite::isCache() +{ + exec("pragma synchronous = off"); + exec("pragma main.journal_mode = truncate"); +} + void SQLite::exec(const std::string & stmt) { retrySQLite<void>([&]() { @@ -94,6 +105,16 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool return *this; } +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size_t len, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { @@ -183,7 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; - printError("warning: %s", e.what()); + logWarning({ + .name = "Sqlite busy", + .hint = hintfmt(e.what()) + }); } /* Sleep for a while since retrying the transaction right away diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index bd012d9b9..dd81ab051 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -3,10 +3,10 @@ #include <functional> #include <string> -#include "types.hh" +#include "error.hh" -class sqlite3; -class sqlite3_stmt; +struct sqlite3; +struct sqlite3_stmt; namespace nix { @@ -15,13 +15,16 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path); + SQLite(const Path & path, bool create = true); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } ~SQLite(); operator sqlite3 * () { return db; } + /* Disable synchronous mode, set truncate journal mode. */ + void isCache(); + void exec(const std::string & stmt); }; @@ -52,6 +55,7 @@ struct SQLiteStmt /* Bind the next parameter. */ Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); Use & bind(); // null diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b043feb0a..aae227bae 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,6 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "url.hh" #include <future> @@ -22,7 +23,7 @@ bool Store::isInStore(const Path & path) const Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return path; @@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } @@ -54,15 +55,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const { auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs)); -} - - -string storePathToHash(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() >= storePathHashLen); - return string(base, 0, storePathHashLen); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; } @@ -141,9 +134,9 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); - return StorePath::make(h.hash, name); + return StorePath(h, name); } @@ -171,19 +164,22 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - bool recursive, + FileIngestionMethod recursive, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive) { + if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); - return makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), name); + return makeStorePath("output:out", + hashString(htSHA256, + "fixed:out:" + + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + hash.to_string(Base16, true) + ":"), + name); } } @@ -200,10 +196,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const + const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(recursive, h, name), h); + Hash h = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + return std::make_pair(makeFixedOutputPath(method, h, name), h); } @@ -237,7 +235,7 @@ bool Store::PathInfoCacheValue::isKnownNow() bool Store::isValidPath(const StorePath & storePath) { - auto hashPart = storePathToHash(printStorePath(storePath)); + std::string hashPart(storePath.hashPart()); { auto state_(state.lock()); @@ -305,7 +303,7 @@ void Store::queryPathInfo(const StorePath & storePath, std::string hashPart; try { - hashPart = storePathToHash(printStorePath(storePath)); + hashPart = storePath.hashPart(); { auto res = state.lock()->pathInfoCache.get(hashPart); @@ -441,7 +439,9 @@ string Store::makeValidityRegistration(const StorePathSet & paths, void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) + bool includeImpureInfo, bool showClosureSize, + Base hashBase, + AllowInvalidFlag allowInvalid) { auto jsonList = jsonOut.list(); @@ -453,7 +453,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string()) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -496,7 +496,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -545,7 +545,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) unsupported("buildPaths"); - paths2.insert(path.path.clone()); + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) @@ -685,21 +685,6 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, } -ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) - : path(other.path.clone()) - , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{}) - , narHash(other.narHash) - , references(cloneStorePathSet(other.references)) - , registrationTime(other.registrationTime) - , narSize(other.narSize) - , id(other.id) - , ultimate(other.ultimate) - , sigs(other.sigs) - , ca(other.ca) -{ -} - - std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) { std::string path; @@ -741,12 +726,7 @@ std::string Store::showPaths(const StorePathSet & paths) string showPaths(const PathSet & paths) { - string s; - for (auto & i : paths) { - if (s.size() != 0) s += ", "; - s += "'" + i + "'"; - } - return s; + return concatStringsSep(", ", quoteStrings(paths)); } @@ -757,7 +737,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -772,11 +752,15 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) bool ValidPathInfo::isContentAddressed(const Store & store) const { auto warn = [&]() { - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + logWarning( + ErrorInfo{ + .name = "Path not content-addressed", + .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) + }); }; if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); + Hash hash(ca.substr(5)); if (store.makeTextPath(path.name(), hash, references) == path) return true; else @@ -784,9 +768,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - Hash hash(std::string(ca, recursive ? 8 : 6)); - auto refs = cloneStorePathSet(references); + FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; + Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); + auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; @@ -829,26 +813,13 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(bool recursive, const Hash & hash) -{ - return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); -} - - -void Store::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) +std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { - addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor); + return "fixed:" + + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + + hash.to_string(Base32, true); } -void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - StringSource source(*nar); - addToStore(info, source, repair, checkSigs, accessor); -} } @@ -869,27 +840,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; - } - } + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; @@ -964,7 +915,7 @@ std::list<ref<Store>> getDefaultSubstituters() try { stores.push_back(openStore(uri)); } catch (Error & e) { - printError("warning: %s", e.what()); + logWarning(e.info()); } }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e0484de13..6f4dd959c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -28,6 +28,7 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); +MakeError(NotInStore, Error); struct BasicDerivation; @@ -43,7 +44,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; @@ -189,8 +189,9 @@ struct ValidPathInfo Strings shortRefs() const; + ValidPathInfo(const StorePath & path) : path(path) { } + ValidPathInfo(StorePath && path) : path(std::move(path)) { } - explicit ValidPathInfo(const ValidPathInfo & other); virtual ~ValidPathInfo() { } }; @@ -346,7 +347,7 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(bool recursive, + StorePath makeFixedOutputPath(FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references = {}, bool hasSelfReference = false) const; @@ -358,7 +359,7 @@ public: store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive = true, + const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -430,10 +431,6 @@ public: virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const StorePath & path) - { unsupported("queryDerivationOutputNames"); } - /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; @@ -450,24 +447,19 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); - - // FIXME: remove - virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); + std::shared_ptr<FSAccessor> accessor = 0) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -561,6 +553,7 @@ public: each path is included. */ void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, + Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, @@ -591,6 +584,9 @@ public: ensurePath(). */ Derivation derivationFromPath(const StorePath & drvPath); + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If @@ -676,6 +672,11 @@ public: return storePath; } + Path toRealPath(const StorePath & storePath) + { + return toRealPath(printStorePath(storePath)); + } + virtual void createUser(const std::string & userName, uid_t userId) { } @@ -731,10 +732,6 @@ public: }; -/* Extract the hash part of the given store path. */ -string storePathToHash(const Path & path); - - /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); @@ -844,7 +841,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo( /* Compute the content-addressability assertion (ValidPathInfo::ca) for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(bool recursive, const Hash & hash); +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); /* Split URI into protocol+hierarchy part and its parameter set. */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 857d54d99..ac42457fc 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,7 +36,7 @@ typedef enum { wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, + wopQueryDerivationOutputNames = 28, // obsolete wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc index 98f8287ad..ac2295e4a 100644 --- a/src/libutil/affinity.cc +++ b/src/libutil/affinity.cc @@ -12,6 +12,17 @@ namespace nix { #if __linux__ static bool didSaveAffinity = false; static cpu_set_t savedAffinity; + +std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset) +{ + auto count = CPU_COUNT(&cset); + for (int i=0; i < count; ++i) + { + os << (CPU_ISSET(i,&cset) ? "1" : "0"); + } + + return os; +} #endif @@ -25,7 +36,7 @@ void setAffinityTo(int cpu) CPU_ZERO(&newAffinity); CPU_SET(cpu, &newAffinity); if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printError(format("failed to lock thread to CPU %1%") % cpu); + printError("failed to lock thread to CPU %1%", cpu); #endif } @@ -47,7 +58,11 @@ void restoreAffinity() #if __linux__ if (!didSaveAffinity) return; if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - printError("failed to restore affinity %1%"); + { + std::ostringstream oss; + oss << savedAffinity; + printError("failed to restore CPU affinity %1%", oss.str()); + } #endif } diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh new file mode 100644 index 000000000..8ae07b092 --- /dev/null +++ b/src/libutil/ansicolor.hh @@ -0,0 +1,15 @@ +#pragma once + +namespace nix { + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_ITALIC "\e[3m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_YELLOW "\e[33;1m" +#define ANSI_BLUE "\e[34;1m" + +} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index db544a212..6a8484705 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -46,7 +46,7 @@ static void dumpContents(const Path & path, size_t size, sink << "contents" << size; AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); + if (!fd) throw SysError("opening file '%1%'", path); std::vector<unsigned char> buf(65536); size_t left = size; @@ -68,7 +68,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); sink << "("; @@ -94,8 +94,9 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) name.erase(pos); } if (unhacked.find(name) != unhacked.end()) - throw Error(format("file name collision in between '%1%' and '%2%'") - % (path + "/" + unhacked[name]) % (path + "/" + i.name)); + throw Error("file name collision in between '%1%' and '%2%'", + (path + "/" + unhacked[name]), + (path + "/" + i.name)); unhacked[name] = i.name; } else unhacked[i.name] = i.name; @@ -111,7 +112,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else if (S_ISLNK(st.st_mode)) sink << "type" << "symlink" << "target" << readLink(path); - else throw Error(format("file '%1%' has an unsupported type") % path); + else throw Error("file '%1%' has an unsupported type", path); sink << ")"; } @@ -247,7 +248,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "name") { name = readString(source); if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) - throw Error(format("NAR contains invalid file name '%1%'") % name); + throw Error("NAR contains invalid file name '%1%'", name); if (name <= prevName) throw Error("NAR directory is not sorted"); prevName = name; @@ -303,14 +304,14 @@ struct RestoreSink : ParseSink { Path p = dstPath + path; if (mkdir(p.c_str(), 0777) == -1) - throw SysError(format("creating directory '%1%'") % p); + throw SysError("creating directory '%1%'", p); }; void createRegularFile(const Path & path) { Path p = dstPath + path; fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError(format("creating file '%1%'") % p); + if (!fd) throw SysError("creating file '%1%'", p); } void isExecutable() @@ -332,7 +333,7 @@ struct RestoreSink : ParseSink OpenSolaris). Since preallocation is just an optimisation, ignore it. */ if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError(format("preallocating file of %1% bytes") % len); + throw SysError("preallocating file of %1% bytes", len); } #endif } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ba15ea571..ce6580119 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -3,16 +3,14 @@ namespace nix { -Args::FlagMaker Args::mkFlag() -{ - return FlagMaker(*this); -} - -Args::FlagMaker::~FlagMaker() +void Args::addFlag(Flag && flag_) { + auto flag = std::make_shared<Flag>(std::move(flag_)); + if (flag->handler.arity != ArityAny) + assert(flag->handler.arity == flag->labels.size()); assert(flag->longName != ""); - args.longFlags[flag->longName] = flag; - if (flag->shortName) args.shortFlags[flag->shortName] = flag; + longFlags[flag->longName] = flag; + if (flag->shortName) shortFlags[flag->shortName] = flag; } void Args::parseCmdline(const Strings & _cmdline) @@ -47,7 +45,7 @@ void Args::parseCmdline(const Strings & _cmdline) } else if (!dashDash && std::string(arg, 0, 1) == "-") { if (!processFlag(pos, cmdline.end())) - throw UsageError(format("unrecognised flag '%1%'") % arg); + throw UsageError("unrecognised flag '%1%'", arg); } else { pendingArgs.push_back(*pos++); @@ -61,7 +59,7 @@ void Args::parseCmdline(const Strings & _cmdline) void Args::printHelp(const string & programName, std::ostream & out) { - std::cout << "Usage: " << programName << " <FLAGS>..."; + std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName); for (auto & exp : expectedArgs) { std::cout << renderLabels({exp.label}); // FIXME: handle arity > 1 @@ -72,11 +70,11 @@ void Args::printHelp(const string & programName, std::ostream & out) auto s = description(); if (s != "") - std::cout << "\nSummary: " << s << ".\n"; + std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n"; if (longFlags.size()) { std::cout << "\n"; - std::cout << "Flags:\n"; + std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n"; printFlags(out); } } @@ -101,15 +99,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; std::vector<std::string> args; - for (size_t n = 0 ; n < flag.arity; ++n) { + for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { - if (flag.arity == ArityAny) break; - throw UsageError(format("flag '%1%' requires %2% argument(s)") - % name % flag.arity); + if (flag.handler.arity == ArityAny) break; + throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } args.push_back(*pos++); } - flag.handler(std::move(args)); + flag.handler.fun(std::move(args)); return true; }; @@ -133,7 +130,7 @@ bool Args::processArgs(const Strings & args, bool finish) { if (expectedArgs.empty()) { if (!args.empty()) - throw UsageError(format("unexpected argument '%1%'") % args.front()); + throw UsageError("unexpected argument '%1%'", args.front()); return true; } @@ -157,17 +154,18 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } -Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) +Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) { - arity(1); - label("type"); - description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); - handler([ht](std::string s) { - *ht = parseHashType(s); - if (*ht == htUnknown) - throw UsageError("unknown hash type '%1%'", s); - }); - return *this; + return Flag { + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError("unknown hash type '%1%'", s); + }} + }; } Strings argvToStrings(int argc, char * * argv) @@ -183,7 +181,7 @@ std::string renderLabels(const Strings & labels) std::string res; for (auto label : labels) { for (auto & c : label) c = std::toupper(c); - res += " <" + label + ">"; + res += " " ANSI_ITALIC + label + ANSI_NORMAL; } return res; } @@ -192,10 +190,10 @@ void printTable(std::ostream & out, const Table2 & table) { size_t max = 0; for (auto & row : table) - max = std::max(max, row.first.size()); + max = std::max(max, filterANSIEscapes(row.first, true).size()); for (auto & row : table) { out << " " << row.first - << std::string(max - row.first.size() + 2, ' ') + << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ') << row.second << "\n"; } } @@ -206,8 +204,7 @@ void Command::printHelp(const string & programName, std::ostream & out) auto exs = examples(); if (!exs.empty()) { - out << "\n"; - out << "Examples:\n"; + out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; for (auto & ex : exs) out << "\n" << " " << ex.description << "\n" // FIXME: wrap @@ -220,52 +217,63 @@ MultiCommand::MultiCommand(const Commands & commands) { expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) { assert(!command); - auto i = commands.find(ss[0]); + auto cmd = ss[0]; + if (auto alias = get(deprecatedAliases, cmd)) { + warn("'%s' is a deprecated alias for '%s'", cmd, *alias); + cmd = *alias; + } + auto i = commands.find(cmd); if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", ss[0]); - command = i->second(); - command->_name = ss[0]; + throw UsageError("'%s' is not a recognised command", cmd); + command = {cmd, i->second()}; }}); + + categories[Command::catDefault] = "Available commands"; } void MultiCommand::printHelp(const string & programName, std::ostream & out) { if (command) { - command->printHelp(programName + " " + command->name(), out); + command->second->printHelp(programName + " " + command->first, out); return; } - out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; + out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName); - out << "\n"; - out << "Common flags:\n"; + out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n"; printFlags(out); - out << "\n"; - out << "Available commands:\n"; + std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory; - Table2 table; - for (auto & i : commands) { - auto command = i.second(); - command->_name = i.first; - auto descr = command->description(); - if (!descr.empty()) - table.push_back(std::make_pair(command->name(), descr)); + for (auto & [name, commandFun] : commands) { + auto command = commandFun(); + commandsByCategory[command->category()].insert_or_assign(name, command); + } + + for (auto & [category, commands] : commandsByCategory) { + out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]); + + Table2 table; + for (auto & [name, command] : commands) { + auto descr = command->description(); + if (!descr.empty()) + table.push_back(std::make_pair(name, descr)); + } + printTable(out, table); } - printTable(out, table); } bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) { if (Args::processFlag(pos, end)) return true; - if (command && command->processFlag(pos, end)) return true; + if (command && command->second->processFlag(pos, end)) return true; return false; } bool MultiCommand::processArgs(const Strings & args, bool finish) { if (command) - return command->processArgs(args, finish); + return command->second->processArgs(args, finish); else return Args::processArgs(args, finish); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 967efbe1c..154d1e6aa 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -32,13 +32,59 @@ protected: struct Flag { typedef std::shared_ptr<Flag> ptr; + + struct Handler + { + std::function<void(std::vector<std::string>)> fun; + size_t arity; + + Handler() {} + + Handler(std::function<void(std::vector<std::string>)> && fun) + : fun(std::move(fun)) + , arity(ArityAny) + { } + + Handler(std::function<void()> && handler) + : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); }) + , arity(0) + { } + + Handler(std::function<void(std::string)> && handler) + : fun([handler{std::move(handler)}](std::vector<std::string> ss) { + handler(std::move(ss[0])); + }) + , arity(1) + { } + + Handler(std::function<void(std::string, std::string)> && handler) + : fun([handler{std::move(handler)}](std::vector<std::string> ss) { + handler(std::move(ss[0]), std::move(ss[1])); + }) + , arity(2) + { } + + template<class T> + Handler(T * dest) + : fun([=](std::vector<std::string> ss) { *dest = ss[0]; }) + , arity(1) + { } + + template<class T> + Handler(T * dest, const T & val) + : fun([=](std::vector<std::string> ss) { *dest = val; }) + , arity(0) + { } + }; + std::string longName; char shortName = 0; std::string description; - Strings labels; - size_t arity = 0; - std::function<void(std::vector<std::string>)> handler; std::string category; + Strings labels; + Handler handler; + + static Flag mkHashTypeFlag(std::string && longName, HashType * ht); }; std::map<std::string, Flag::ptr> longFlags; @@ -65,49 +111,7 @@ protected: public: - class FlagMaker - { - Args & args; - Flag::ptr flag; - friend class Args; - FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { } - public: - ~FlagMaker(); - FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; } - FlagMaker & shortName(char s) { flag->shortName = s; return *this; } - FlagMaker & description(const std::string & s) { flag->description = s; return *this; } - FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; } - FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; } - FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; } - FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; } - FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; } - FlagMaker & handler(std::function<void(std::string)> handler) { - flag->arity = 1; - flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); }; - return *this; - } - FlagMaker & category(const std::string & s) { flag->category = s; return *this; } - - template<class T> - FlagMaker & dest(T * dest) - { - flag->arity = 1; - flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; - return *this; - } - - template<class T> - FlagMaker & set(T * dest, const T & val) - { - flag->arity = 0; - flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; - return *this; - } - - FlagMaker & mkHashTypeFlag(HashType * ht); - }; - - FlagMaker mkFlag(); + void addFlag(Flag && flag); /* Helper functions for constructing flags / positional arguments. */ @@ -116,13 +120,13 @@ public: const std::string & label, const std::string & description, std::function<void(std::string)> fun) { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({label}) - .description(description) - .arity(1) - .handler([=](std::vector<std::string> ss) { fun(ss[0]); }); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {label}, + .handler = {[=](std::string s) { fun(s); }} + }); } void mkFlag(char shortName, const std::string & name, @@ -135,11 +139,12 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, T * dest, const T & value) { - mkFlag() - .shortName(shortName) - .longName(longName) - .description(description) - .handler([=](std::vector<std::string> ss) { *dest = value; }); + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .handler = {[=]() { *dest = value; }} + }); } template<class I> @@ -155,18 +160,18 @@ public: void mkFlag(char shortName, const std::string & longName, const std::string & description, std::function<void(I)> fun) { - mkFlag() - .shortName(shortName) - .longName(longName) - .labels({"N"}) - .description(description) - .arity(1) - .handler([=](std::vector<std::string> ss) { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"N"}, + .handler = {[=](std::string s) { I n; - if (!string2Int(ss[0], n)) + if (!string2Int(s, n)) throw UsageError("flag '--%s' requires a integer argument", longName); fun(n); - }); + }} + }); } /* Expect a string argument. */ @@ -192,17 +197,10 @@ public: run() method. */ struct Command : virtual Args { -private: - std::string _name; - friend class MultiCommand; -public: - virtual ~Command() { } - std::string name() { return _name; } - virtual void prepare() { }; virtual void run() = 0; @@ -216,6 +214,12 @@ public: virtual Examples examples() { return Examples(); } + typedef int Category; + + static constexpr Category catDefault = 0; + + virtual Category category() { return catDefault; } + void printHelp(const string & programName, std::ostream & out) override; }; @@ -228,7 +232,12 @@ class MultiCommand : virtual Args public: Commands commands; - std::shared_ptr<Command> command; + std::map<Command::Category, std::string> categories; + + std::map<std::string, std::string> deprecatedAliases; + + // Selected command, if any. + std::optional<std::pair<std::string, ref<Command>>> command; MultiCommand(const Commands & commands); diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 860b04adb..a117ddc72 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -481,7 +481,7 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next else if (method == "br") return make_ref<BrotliCompressionSink>(nextSink); else - throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); + throw UnknownCompressionMethod("unknown compression method '%s'", method); } ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7551d97d1..8fc700a2b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -65,60 +65,63 @@ void Config::getSettings(std::map<std::string, SettingInfo> & res, bool override res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - string contents = readFile(path); - - unsigned int pos = 0; - - while (pos < contents.size()) { - string line; - while (pos < contents.size() && contents[pos] != '\n') - line += contents[pos++]; - pos++; - - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); - - vector<string> tokens = tokenizeString<vector<string> >(line); - if (tokens.empty()) continue; +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector<string> tokens = tokenizeString<vector<string> >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } - if (tokens.size() < 2) + if (include) { + if (tokens.size() != 2) throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - - auto include = false; - auto ignoreMissing = false; - if (tokens[0] == "include") - include = true; - else if (tokens[0] == "!include") { - include = true; - ignoreMissing = true; + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); } + continue; + } - if (include) { - if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - auto p = absPath(tokens[1], dirOf(path)); - if (pathExists(p)) { - applyConfigFile(p); - } else if (!ignoreMissing) { - throw Error("file '%1%' included from '%2%' not found", p, path); - } - continue; - } + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + string name = tokens[0]; - string name = tokens[0]; + vector<string>::iterator i = tokens.begin(); + advance(i, 2); - vector<string>::iterator i = tokens.begin(); - advance(i, 2); + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; +} - set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow - }; +void AbstractConfig::applyConfigFile(const Path & path) +{ + try { + string contents = readFile(path); + applyConfig(contents, path); } catch (SysError &) { } } @@ -177,12 +180,13 @@ void BaseSetting<T>::toJSON(JSONPlaceholder & out) template<typename T> void BaseSetting<T>::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description(description) - .arity(1) - .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); }) - .category(category); + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .labels = {"value"}, + .handler = {[=](std::string s) { overriden = true; set(s); }}, + }); } template<> void BaseSetting<std::string>::set(const std::string & str) @@ -227,16 +231,18 @@ template<> std::string BaseSetting<bool>::to_string() const template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description(description) - .handler([=](std::vector<std::string> ss) { override(true); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description(description) - .handler([=](std::vector<std::string> ss) { override(false); }) - .category(category); + args.addFlag({ + .longName = name, + .description = description, + .category = category, + .handler = {[=]() { override(true); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = description, + .category = category, + .handler = {[=]() { override(false); }} + }); } template<> void BaseSetting<Strings>::set(const std::string & str) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 7ea78fdaf..66073546e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -1,3 +1,4 @@ +#include <cassert> #include <map> #include <set> @@ -7,6 +8,38 @@ namespace nix { +/** + * The Config class provides Nix runtime configurations. + * + * What is a Configuration? + * A collection of uniquely named Settings. + * + * What is a Setting? + * Each property that you can set in a configuration corresponds to a + * `Setting`. A setting records value and description of a property + * with a default and optional aliases. + * + * A valid configuration consists of settings that are registered to a + * `Config` object instance: + * + * Config config; + * Setting<std::string> systemSetting{&config, "x86_64-linux", "system", "the current system"}; + * + * The above creates a `Config` object and registers a setting called "system" + * via the variable `systemSetting` with it. The setting defaults to the string + * "x86_64-linux", it's description is "the current system". All of the + * registered settings can then be accessed as shown below: + * + * std::map<std::string, Config::SettingInfo> settings; + * config.getSettings(settings); + * config["system"].description == "the current system" + * config["system"].value == "x86_64-linux" + * + * + * The above retrieves all currently known settings from the `Config` object + * and adds them to the `settings` map. + */ + class Args; class AbstractSetting; class JSONPlaceholder; @@ -23,6 +56,10 @@ protected: public: + /** + * Sets the value referenced by `name` to `value`. Returns true if the + * setting is known, false otherwise. + */ virtual bool set(const std::string & name, const std::string & value) = 0; struct SettingInfo @@ -31,18 +68,52 @@ public: std::string description; }; + /** + * Adds the currently known settings to the given result map `res`. + * - res: map to store settings in + * - overridenOnly: when set to true only overridden settings will be added to `res` + */ virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0; + /** + * Parses the configuration in `contents` and applies it + * - contents: configuration contents to be parsed and applied + * - path: location of the configuration file + */ + void applyConfig(const std::string & contents, const std::string & path = "<unknown>"); + + /** + * Applies a nix configuration file + * - path: the location of the config file to apply + */ void applyConfigFile(const Path & path); + /** + * Resets the `overridden` flag of all Settings + */ virtual void resetOverriden() = 0; + /** + * Outputs all settings to JSON + * - out: JSONObject to write the configuration to + */ virtual void toJSON(JSONObject & out) = 0; + /** + * Converts settings to `Args` to be used on the command line interface + * - args: args to write to + * - category: category of the settings + */ virtual void convertToArgs(Args & args, const std::string & category) = 0; + /** + * Logs a warning for each unregistered setting + */ void warnUnknownSettings(); + /** + * Re-applies all previously attempted changes to unknown settings + */ void reapplyUnknownSettings(); }; diff --git a/src/libutil/error.cc b/src/libutil/error.cc new file mode 100644 index 000000000..0fad9ae42 --- /dev/null +++ b/src/libutil/error.cc @@ -0,0 +1,221 @@ +#include "error.hh" + +#include <iostream> +#include <optional> +#include "serialise.hh" +#include <sstream> + +namespace nix { + + +const std::string nativeSystem = SYSTEM; + +// addPrefix is used for show-trace. Strings added with addPrefix +// will print ahead of the error itself. +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + +// c++ std::exception descendants must have a 'const char* what()' function. +// This stringifies the error and caches it for use by what(), or similarly by msg(). +const string& BaseError::calcWhat() const +{ + if (what_.has_value()) + return *what_; + else { + err.name = sname(); + + std::ostringstream oss; + oss << err; + what_ = oss.str(); + + return *what_; + } +} + +std::optional<string> ErrorInfo::programName = std::nullopt; + +std::ostream& operator<<(std::ostream &os, const hintformat &hf) +{ + return os << hf.str(); +} + +string showErrPos(const ErrPos &errPos) +{ + if (errPos.line > 0) { + if (errPos.column > 0) { + return fmt("(%1%:%2%)", errPos.line, errPos.column); + } else { + return fmt("(%1%)", errPos.line); + } + } + else { + return ""; + } +} + +// if nixCode contains lines of code, print them to the ostream, indicating the error column. +void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode) +{ + // previous line of code. + if (nixCode.prevLineOfCode.has_value()) { + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line - 1), + *nixCode.prevLineOfCode); + } + + if (nixCode.errLineOfCode.has_value()) { + // line of code containing the error. + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line), + *nixCode.errLineOfCode); + // error arrows for the column range. + if (nixCode.errPos.column > 0) { + int start = nixCode.errPos.column; + std::string spaces; + for (int i = 0; i < start; ++i) { + spaces.append(" "); + } + + std::string arrows("^"); + + out << std::endl + << fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL, + prefix, + spaces, + arrows); + } + } + + // next line of code. + if (nixCode.nextLineOfCode.has_value()) { + out << std::endl + << fmt("%1% %|2$5d|| %3%", + prefix, + (nixCode.errPos.line + 1), + *nixCode.nextLineOfCode); + } +} + +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo) +{ + auto errwidth = std::max<size_t>(getWindowSize().second, 20); + string prefix = ""; + + string levelString; + switch (einfo.level) { + case Verbosity::lvlError: { + levelString = ANSI_RED; + levelString += "error:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlWarn: { + levelString = ANSI_YELLOW; + levelString += "warning:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlInfo: { + levelString = ANSI_GREEN; + levelString += "info:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlTalkative: { + levelString = ANSI_GREEN; + levelString += "talk:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlChatty: { + levelString = ANSI_GREEN; + levelString += "chat:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlVomit: { + levelString = ANSI_GREEN; + levelString += "vomit:"; + levelString += ANSI_NORMAL; + break; + } + case Verbosity::lvlDebug: { + levelString = ANSI_YELLOW; + levelString += "debug:"; + levelString += ANSI_NORMAL; + break; + } + default: { + levelString = fmt("invalid error level: %1%", einfo.level); + break; + } + } + + auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length(); + auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl; + + std::string dashes(dashwidth, '-'); + + // divider. + if (einfo.name != "") + out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL, + prefix, + levelString, + einfo.name, + dashes, + einfo.programName.value_or("")); + else + out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL, + prefix, + levelString, + dashes, + einfo.programName.value_or("")); + + bool nl = false; // intersperse newline between sections. + if (einfo.nixCode.has_value()) { + if (einfo.nixCode->errPos.file != "") { + // filename, line, column. + out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL, + prefix, + einfo.nixCode->errPos.file, + showErrPos(einfo.nixCode->errPos)); + } else { + out << std::endl << fmt("%1%from command line argument", prefix); + } + nl = true; + } + + // description + if (einfo.description != "") { + if (nl) + out << std::endl << prefix; + out << std::endl << prefix << einfo.description; + nl = true; + } + + // lines of code. + if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) { + if (nl) + out << std::endl << prefix; + printCodeLines(out, prefix, *einfo.nixCode); + nl = true; + } + + // hint + if (einfo.hint.has_value()) { + if (nl) + out << std::endl << prefix; + out << std::endl << prefix << *einfo.hint; + nl = true; + } + + return out; +} +} diff --git a/src/libutil/error.hh b/src/libutil/error.hh new file mode 100644 index 000000000..1e6102ce1 --- /dev/null +++ b/src/libutil/error.hh @@ -0,0 +1,186 @@ +#pragma once + + +#include "ref.hh" +#include "types.hh" + +#include <cstring> +#include <list> +#include <memory> +#include <map> +#include <optional> + +#include "fmt.hh" + +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + +namespace nix { + +/* + +This file defines two main structs/classes used in nix error handling. + +ErrorInfo provides a standard payload of error information, with conversion to string +happening in the logger rather than at the call site. + +BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains +an ErrorInfo. + +ErrorInfo structs are sent to the logger as part of an exception, or directly with the +logError or logWarning macros. + +See the error-demo.cc program for usage examples. + +*/ + +typedef enum { + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +// ErrPos indicates the location of an error in a nix file. +struct ErrPos { + int line = 0; + int column = 0; + string file; + + operator bool() const + { + return line != 0; + } + + // convert from the Pos struct, found in libexpr. + template <class P> + ErrPos& operator=(const P &pos) + { + line = pos.line; + column = pos.column; + file = pos.file; + return *this; + } + + template <class P> + ErrPos(const P &p) + { + *this = p; + } +}; + +struct NixCode { + ErrPos errPos; + std::optional<string> prevLineOfCode; + std::optional<string> errLineOfCode; + std::optional<string> nextLineOfCode; +}; + +struct ErrorInfo { + Verbosity level; + string name; + string description; + std::optional<hintformat> hint; + std::optional<NixCode> nixCode; + + static std::optional<string> programName; +}; + +std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo); + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + mutable ErrorInfo err; + + mutable std::optional<string> what_; + const string& calcWhat() const; + +public: + unsigned int status = 1; // exit status + + template<typename... Args> + BaseError(unsigned int status, const Args & ... args) + : err { .level = lvlError, + .hint = hintfmt(args...) + } + , status(status) + { } + + template<typename... Args> + BaseError(const std::string & fs, const Args & ... args) + : err { .level = lvlError, + .hint = hintfmt(fs, args...) + } + { } + + BaseError(hintformat hint) + : err { .level = lvlError, + .hint = hint + } + { } + + BaseError(ErrorInfo && e) + : err(std::move(e)) + { } + + BaseError(const ErrorInfo & e) + : err(e) + { } + + virtual const char* sname() const { return "BaseError"; } + +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~BaseError() throw () { }; + const char * what() const throw () { return calcWhat().c_str(); } +#else + const char * what() const noexcept override { return calcWhat().c_str(); } +#endif + + const string & msg() const { return calcWhat(); } + const string & prefix() const { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); + + const ErrorInfo & info() { calcWhat(); return err; } +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + using superClass::superClass; \ + virtual const char* sname() const override { return #newClass; } \ + } + +MakeError(Error, BaseError); + +class SysError : public Error +{ +public: + int errNo; + + template<typename... Args> + SysError(const Args & ... args) + :Error("") + { + errNo = errno; + auto hf = hintfmt(args...); + err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + } + + virtual const char* sname() const override { return "SysError"; } +}; + +} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh new file mode 100644 index 000000000..12ab9c407 --- /dev/null +++ b/src/libutil/fmt.hh @@ -0,0 +1,139 @@ +#pragma once + +#include <boost/format.hpp> +#include <string> +#include "ansicolor.hh" + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using boost::format; + + +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template<typename... T> nop(T...) {} }; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + template<class F> + FormatOrString(const F & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + equivalent to ‘boost::format(format) % a_0 % ... % + ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + takes place). */ + +template<class F> +inline void formatHelper(F & f) +{ +} + +template<class F, typename T, typename... Args> +inline void formatHelper(F & f, const T & x, const Args & ... args) +{ + formatHelper(f % x, args...); +} + +inline std::string fmt(const std::string & s) +{ + return s; +} + +inline std::string fmt(const char * s) +{ + return s; +} + +inline std::string fmt(const FormatOrString & fs) +{ + return fs.s; +} + +template<typename... Args> +inline std::string fmt(const std::string & fs, const Args & ... args) +{ + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + formatHelper(f, args...); + return f.str(); +} + +// ----------------------------------------------------------------------------- +// format function for hints in errors. same as fmt, except templated values +// are always in yellow. + +template <class T> +struct yellowtxt +{ + yellowtxt(const T &s) : value(s) {} + const T &value; +}; + +template <class T> +std::ostream& operator<<(std::ostream &out, const yellowtxt<T> &y) +{ + return out << ANSI_YELLOW << y.value << ANSI_NORMAL; +} + +template <class T> +struct normaltxt +{ + normaltxt(const T &s) : value(s) {} + const T &value; +}; + +template <class T> +std::ostream& operator<<(std::ostream &out, const normaltxt<T> &y) +{ + return out << ANSI_NORMAL << y.value; +} + +class hintformat +{ +public: + hintformat(const string &format) :fmt(format) + { + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + } + + hintformat(const hintformat &hf) + : fmt(hf.fmt) + {} + + template<class T> + hintformat& operator%(const T &value) + { + fmt % yellowtxt(value); + return *this; + } + + std::string str() const + { + return fmt.str(); + } + +private: + format fmt; +}; + +std::ostream& operator<<(std::ostream &os, const hintformat &hf); + +template<typename... Args> +inline hintformat hintfmt(const std::string & fs, const Args & ... args) +{ + hintformat f(fs); + formatHelper(f, args...); + return f; +} + +} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 7caee1da7..460d479a3 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -125,7 +125,7 @@ std::string Hash::to_string(Base base, bool includeType) const } -Hash::Hash(const std::string & s, HashType type) +Hash::Hash(std::string_view s, HashType type) : type(type) { size_t pos = 0; @@ -194,7 +194,7 @@ Hash::Hash(const std::string & s, HashType type) } else if (isSRI || size == base64Len()) { - auto d = base64Decode(std::string(s, pos)); + auto d = base64Decode(s.substr(pos)); if (d.size() != hashSize) throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); assert(hashSize); @@ -205,6 +205,16 @@ Hash::Hash(const std::string & s, HashType type) throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); } +Hash newHashAllowEmpty(std::string hashStr, HashType ht) +{ + if (hashStr.empty()) { + Hash h(ht); + warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + return h; + } else + return Hash(hashStr, ht); +} + union Ctx { diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ffa43ecf5..180fb7633 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -42,7 +42,7 @@ struct Hash Subresource Integrity hash expression). If the 'type' argument is htUnknown, then the hash type must be specified in the string. */ - Hash(const std::string & s, HashType type = htUnknown); + Hash(std::string_view s, HashType type = htUnknown); void init(); @@ -79,9 +79,23 @@ struct Hash /* Return a string representation of the hash, in base-16, base-32 or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ - std::string to_string(Base base = Base32, bool includeType = true) const; + std::string to_string(Base base, bool includeType) const; + + std::string gitRev() const + { + assert(type == htSHA1); + return to_string(Base16, false); + } + + std::string gitShortRev() const + { + assert(type == htSHA1); + return std::string(to_string(Base16, false), 0, 7); + } }; +/* Helper that defaults empty hashes to the 0 hash. */ +Hash newHashAllowEmpty(std::string hashStr, HashType ht); /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ string printHash16or32(const Hash & hash); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 16c1fa03f..ae7eb67ad 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -7,5 +7,3 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context - -libutil_LIBS = libnixrust diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index fa5c84a27..105fadb15 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,6 +3,7 @@ #include <atomic> #include <nlohmann/json.hpp> +#include <iostream> namespace nix { @@ -17,25 +18,36 @@ void setCurActivity(const ActivityId activityId) curActivity = activityId; } -Logger * logger = makeDefaultLogger(); +Logger * logger = makeSimpleLogger(true); void Logger::warn(const std::string & msg) { log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg); } +void Logger::writeToStdout(std::string_view s) +{ + std::cout << s << "\n"; +} + class SimpleLogger : public Logger { public: bool systemd, tty; + bool printBuildLogs; - SimpleLogger() + SimpleLogger(bool printBuildLogs) + : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; tty = isatty(STDERR_FILENO); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { if (lvl > verbosity) return; @@ -57,13 +69,33 @@ public: writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); } + void logEI(const ErrorInfo & ei) override + { + std::stringstream oss; + oss << ei; + + log(ei.level, oss.str()); + } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) - override + override { if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (type == resBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError(lastLine); + } + else if (type == resPostBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError("post-build-hook: " + lastLine); + } + } }; Verbosity verbosity = lvlInfo; @@ -88,9 +120,9 @@ void writeToStderr(const string & s) } } -Logger * makeDefaultLogger() +Logger * makeSimpleLogger(bool printBuildLogs) { - return new SimpleLogger(); + return new SimpleLogger(printBuildLogs); } std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32}; @@ -102,12 +134,15 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, logger.startActivity(id, lvl, type, s, fields, parent); } -struct JSONLogger : Logger -{ +struct JSONLogger : Logger { Logger & prevLogger; JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + bool isVerbose() override { + return true; + } + void addFields(nlohmann::json & json, const Fields & fields) { if (fields.empty()) return; @@ -135,6 +170,19 @@ struct JSONLogger : Logger write(json); } + void logEI(const ErrorInfo & ei) override + { + std::ostringstream oss; + oss << ei; + + nlohmann::json json; + json["action"] = "msg"; + json["level"] = ei.level; + json["msg"] = oss.str(); + + write(json); + } + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, const std::string & s, const Fields & fields, ActivityId parent) override { @@ -198,7 +246,7 @@ bool handleJSONLogMessage(const std::string & msg, if (action == "start") { auto type = (ActivityType) json["type"]; - if (trusted || type == actDownload) + if (trusted || type == actFileTransfer) activities.emplace(std::piecewise_construct, std::forward_as_tuple(json["id"]), std::forward_as_tuple(*logger, (Verbosity) json["level"], type, @@ -225,13 +273,17 @@ bool handleJSONLogMessage(const std::string & msg, } } catch (std::exception & e) { - printError("bad log message from builder: %s", e.what()); + logError({ + .name = "Json log message", + .hint = hintfmt("bad log message from builder: %s", e.what()) + }); } return true; } -Activity::~Activity() { +Activity::~Activity() +{ try { logger.stopActivity(id); } catch (...) { diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index beb5e6b64..b1583eced 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,23 +1,14 @@ #pragma once #include "types.hh" +#include "error.hh" namespace nix { typedef enum { - lvlError = 0, - lvlWarn, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - -typedef enum { actUnknown = 0, actCopyPath = 100, - actDownload = 101, + actFileTransfer = 101, actRealise = 102, actCopyPaths = 103, actBuilds = 104, @@ -27,6 +18,7 @@ typedef enum { actSubstitute = 108, actQueryPathInfo = 109, actPostBuildHook = 110, + actBuildWaiting = 111, } ActivityType; typedef enum { @@ -63,6 +55,11 @@ public: virtual ~Logger() { } + virtual void stop() { }; + + // Whether the logger prints the whole build log + virtual bool isVerbose() { return false; } + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; void log(const FormatOrString & fs) @@ -70,6 +67,14 @@ public: log(lvlInfo, fs); } + virtual void logEI(const ErrorInfo &ei) = 0; + + void logEI(Verbosity lvl, ErrorInfo ei) + { + ei.level = lvl; + logEI(ei); + } + virtual void warn(const std::string & msg); virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -78,6 +83,16 @@ public: virtual void stopActivity(ActivityId act) { }; virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + + virtual void writeToStdout(std::string_view s); + + template<typename... Args> + inline void stdout(const std::string & fs, const Args & ... args) + { + boost::format f(fs); + formatHelper(f, args...); + writeToStdout(f.str()); + } }; ActivityId getCurActivity(); @@ -131,7 +146,7 @@ struct PushActivity extern Logger * logger; -Logger * makeDefaultLogger(); +Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); @@ -141,9 +156,23 @@ bool handleJSONLogMessage(const std::string & msg, extern Verbosity verbosity; /* suppress msgs > this */ -/* Print a message if the current log level is at least the specified - level. Note that this has to be implemented as a macro to ensure - that the arguments are evaluated lazily. */ +/* Print a message with the standard ErrorInfo format. + In general, use these 'log' macros for reporting problems that may require user + intervention or that need more explanation. Use the 'print' macros for more + lightweight status messages. */ +#define logErrorInfo(level, errorInfo...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->logEI(level, errorInfo); \ + } \ + } while (0) + +#define logError(errorInfo...) logErrorInfo(lvlError, errorInfo) +#define logWarning(errorInfo...) logErrorInfo(lvlWarn, errorInfo) + +/* Print a string message if the current log level is at least the specified + level. Note that this has to be implemented as a macro to ensure that the + arguments are evaluated lazily. */ #define printMsg(level, args...) \ do { \ if (level <= nix::verbosity) { \ @@ -157,6 +186,7 @@ extern Verbosity verbosity; /* suppress msgs > this */ #define debug(args...) printMsg(lvlDebug, args) #define vomit(args...) printMsg(lvlVomit, args) +/* if verbosity >= lvlWarn, print a message with a yellow 'warning:' prefix. */ template<typename... Args> inline void warn(const std::string & fs, const Args & ... args) { diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 8b83f842c..6ef4a3e06 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -1,5 +1,6 @@ #pragma once +#include <cassert> #include <map> #include <list> #include <optional> diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index 6f36b3192..67924568f 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -1,3 +1,4 @@ +#if 0 #include "logging.hh" #include "rust-ffi.hh" @@ -20,3 +21,4 @@ std::ostream & operator << (std::ostream & str, const String & s) } } +#endif diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index 228e2eead..cfbaf9dec 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -1,4 +1,5 @@ #pragma once +#if 0 #include "serialise.hh" @@ -185,3 +186,4 @@ struct Result }; } +#endif diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8201549fd..c8b71188f 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -52,7 +52,10 @@ size_t threshold = 256 * 1024 * 1024; static void warnLargeDump() { - printError("warning: dumping very large path (> 256 MiB); this may run out of memory"); + logWarning({ + .name = "Large path", + .description = "dumping very large path (> 256 MiB); this may run out of memory" + }); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 5780c93a6..a04118512 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -148,6 +148,9 @@ struct StringSink : Sink { ref<std::string> s; StringSink() : s(make_ref<std::string>()) { }; + explicit StringSink(const size_t reservedSize) : s(make_ref<std::string>()) { + s->reserve(reservedSize); + }; StringSink(ref<std::string> s) : s(s) { }; void operator () (const unsigned char * data, size_t len) override; }; diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc new file mode 100644 index 000000000..74c59fd31 --- /dev/null +++ b/src/libutil/tests/config.cc @@ -0,0 +1,264 @@ +#include "json.hh" +#include "config.hh" +#include "args.hh" + +#include <sstream> +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * Config + * --------------------------------------------------------------------------*/ + + TEST(Config, setUndefinedSetting) { + Config config; + ASSERT_EQ(config.set("undefined-key", "value"), false); + } + + TEST(Config, setDefinedSetting) { + Config config; + std::string value; + Setting<std::string> foo{&config, value, "name-of-the-setting", "description"}; + ASSERT_EQ(config.set("name-of-the-setting", "value"), true); + } + + TEST(Config, getDefinedSetting) { + Config config; + std::string value; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, ""); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedOverridenSettingNotSet) { + Config config; + std::string value; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> foo{&config, value, "name-of-the-setting", "description"}; + + config.getSettings(settings, /* overridenOnly = */ true); + const auto e = settings.find("name-of-the-setting"); + ASSERT_EQ(e, settings.end()); + } + + TEST(Config, getDefinedSettingSet1) { + Config config; + std::string value; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, value, "name-of-the-setting", "description"}; + + setting.assign("value"); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto iter = settings.find("name-of-the-setting"); + ASSERT_NE(iter, settings.end()); + ASSERT_EQ(iter->second.value, "value"); + ASSERT_EQ(iter->second.description, "description"); + } + + TEST(Config, getDefinedSettingSet2) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_TRUE(config.set("name-of-the-setting", "value")); + + config.getSettings(settings, /* overridenOnly = */ false); + const auto e = settings.find("name-of-the-setting"); + ASSERT_NE(e, settings.end()); + ASSERT_EQ(e->second.value, "value"); + ASSERT_EQ(e->second.description, "description"); + } + + TEST(Config, addSetting) { + class TestSetting : public AbstractSetting { + public: + TestSetting() : AbstractSetting("test", "test", {}) {} + void set(const std::string & value) {} + std::string to_string() const { return {}; } + }; + + Config config; + TestSetting setting; + + ASSERT_FALSE(config.set("test", "value")); + config.addSetting(&setting); + ASSERT_TRUE(config.set("test", "value")); + } + + TEST(Config, withInitialValue) { + const StringMap initials = { + { "key", "value" }, + }; + Config config(initials); + + { + std::map<std::string, Config::SettingInfo> settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings.find("key"), settings.end()); + } + + Setting<std::string> setting{&config, "default-value", "key", "description"}; + + { + std::map<std::string, Config::SettingInfo> settings; + config.getSettings(settings, /* overridenOnly = */ false); + ASSERT_EQ(settings["key"].value, "value"); + } + } + + TEST(Config, resetOverriden) { + Config config; + config.resetOverriden(); + } + + TEST(Config, resetOverridenWithSetting) { + Config config; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + + { + std::map<std::string, Config::SettingInfo> settings; + + setting.set("foo"); + ASSERT_EQ(setting.get(), "foo"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + + { + std::map<std::string, Config::SettingInfo> settings; + + setting.override("bar"); + ASSERT_TRUE(setting.overriden); + ASSERT_EQ(setting.get(), "bar"); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_FALSE(settings.empty()); + } + + { + std::map<std::string, Config::SettingInfo> settings; + + config.resetOverriden(); + ASSERT_FALSE(setting.overriden); + config.getSettings(settings, /* overridenOnly = */ true); + ASSERT_TRUE(settings.empty()); + } + } + + TEST(Config, toJSONOnEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + Config config; + config.toJSON(obj); + } + + ASSERT_EQ(out.str(), "{}"); + } + + TEST(Config, toJSONOnNonEmptyConfig) { + std::stringstream out; + { // Scoped to force the destructor of JSONObject to write the final `}` + JSONObject obj(out); + + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + setting.assign("value"); + + config.toJSON(obj); + } + ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#"); + } + + TEST(Config, setSettingAlias) { + Config config; + Setting<std::string> setting{&config, "", "some-int", "best number", { "another-int" }}; + ASSERT_TRUE(config.set("some-int", "1")); + ASSERT_EQ(setting.get(), "1"); + ASSERT_TRUE(config.set("another-int", "2")); + ASSERT_EQ(setting.get(), "2"); + ASSERT_TRUE(config.set("some-int", "3")); + ASSERT_EQ(setting.get(), "3"); + } + + /* FIXME: The reapplyUnknownSettings method doesn't seem to do anything + * useful (these days). Whenever we add a new setting to Config the + * unknown settings are always considered. In which case is this function + * actually useful? Is there some way to register a Setting without calling + * addSetting? */ + TEST(Config, DISABLED_reapplyUnknownSettings) { + Config config; + ASSERT_FALSE(config.set("name-of-the-setting", "unknownvalue")); + Setting<std::string> setting{&config, "default", "name-of-the-setting", "description"}; + ASSERT_EQ(setting.get(), "default"); + config.reapplyUnknownSettings(); + ASSERT_EQ(setting.get(), "unknownvalue"); + } + + TEST(Config, applyConfigEmpty) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + config.applyConfig(""); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigEmptyWithComment) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + config.applyConfig("# just a comment"); + config.getSettings(settings); + ASSERT_TRUE(settings.empty()); + } + + TEST(Config, applyConfigAssignment) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = value-from-file #useful comment\n" + "# name-of-the-setting = foo\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "value-from-file"); + } + + TEST(Config, applyConfigWithReassignedSetting) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + config.applyConfig( + "name-of-the-setting = first-value\n" + "name-of-the-setting = second-value\n" + ); + config.getSettings(settings); + ASSERT_FALSE(settings.empty()); + ASSERT_EQ(settings["name-of-the-setting"].value, "second-value"); + } + + TEST(Config, applyConfigFailsOnMissingIncludes) { + Config config; + std::map<std::string, Config::SettingInfo> settings; + Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; + + ASSERT_THROW(config.applyConfig( + "name-of-the-setting = value-from-file\n" + "# name-of-the-setting = foo\n" + "include /nix/store/does/not/exist.nix" + ), Error); + } + + TEST(Config, applyConfigInvalidThrows) { + Config config; + ASSERT_THROW(config.applyConfig("value == key"), UsageError); + ASSERT_THROW(config.applyConfig("value "), UsageError); + } +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc new file mode 100644 index 000000000..5334b046e --- /dev/null +++ b/src/libutil/tests/hash.cc @@ -0,0 +1,80 @@ +#include "hash.hh" +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * hashString + * --------------------------------------------------------------------------*/ + + TEST(hashString, testKnownMD5Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s1 = ""; + auto hash = hashString(HashType::htMD5, s1); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + } + + TEST(hashString, testKnownMD5Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc1321 + auto s2 = "abc"; + auto hash = hashString(HashType::htMD5, s2); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + } + + TEST(hashString, testKnownSHA1Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abc"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + } + + TEST(hashString, testKnownSHA1Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc3174 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA1, s); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + } + + TEST(hashString, testKnownSHA256Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + } + + TEST(hashString, testKnownSHA256Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + auto hash = hashString(HashType::htSHA256, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + } + + TEST(hashString, testKnownSHA512Hashes1) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" + "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" + "454d4423643ce80e2a9ac94fa54ca49f"); + } + + TEST(hashString, testKnownSHA512Hashes2) { + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + + auto hash = hashString(HashType::htSHA512, s); + ASSERT_EQ(hash.to_string(Base::Base16, true), + "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" + "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" + "c7d329eeb6dd26545e96e55b874be909"); + } + + TEST(hashString, hashingWithUnknownAlgoExits) { + auto s = "unknown"; + ASSERT_DEATH(hashString(HashType::htUnknown, s), ""); + } +} diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc new file mode 100644 index 000000000..dea73f53a --- /dev/null +++ b/src/libutil/tests/json.cc @@ -0,0 +1,193 @@ +#include "json.hh" +#include <gtest/gtest.h> +#include <sstream> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * toJSON + * --------------------------------------------------------------------------*/ + + TEST(toJSON, quotesCharPtr) { + const char* input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, quotesStdString) { + std::string input = "test"; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "\"test\""); + } + + TEST(toJSON, convertsNullptrtoNull) { + auto input = nullptr; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + TEST(toJSON, convertsNullToNull) { + const char* input = 0; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "null"); + } + + + TEST(toJSON, convertsFloat) { + auto input = 1.024f; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsDouble) { + const double input = 1.024; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "1.024"); + } + + TEST(toJSON, convertsBool) { + auto input = false; + std::stringstream out; + toJSON(out, input); + + ASSERT_EQ(out.str(), "false"); + } + + TEST(toJSON, quotesTab) { + std::stringstream out; + toJSON(out, "\t"); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + TEST(toJSON, quotesNewline) { + std::stringstream out; + toJSON(out, "\n"); + + ASSERT_EQ(out.str(), "\"\\n\""); + } + + TEST(toJSON, quotesCreturn) { + std::stringstream out; + toJSON(out, "\r"); + + ASSERT_EQ(out.str(), "\"\\r\""); + } + + TEST(toJSON, quotesCreturnNewLine) { + std::stringstream out; + toJSON(out, "\r\n"); + + ASSERT_EQ(out.str(), "\"\\r\\n\""); + } + + TEST(toJSON, quotesDoublequotes) { + std::stringstream out; + toJSON(out, "\""); + + ASSERT_EQ(out.str(), "\"\\\"\""); + } + + TEST(toJSON, substringEscape) { + std::stringstream out; + const char *s = "foo\t"; + toJSON(out, s+3, s + strlen(s)); + + ASSERT_EQ(out.str(), "\"\\t\""); + } + + /* ---------------------------------------------------------------------------- + * JSONObject + * --------------------------------------------------------------------------*/ + + TEST(JSONObject, emptyObject) { + std::stringstream out; + { + JSONObject t(out); + } + ASSERT_EQ(out.str(), "{}"); + } + + TEST(JSONObject, objectWithList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithListIndent) { + std::stringstream out; + { + JSONObject t(out, true); + auto l = t.list("list"); + l.elem("element"); + } + ASSERT_EQ(out.str(), +R"#({ + "list": [ + "element" + ] +})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndList) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("list"); + l.list().elem("element"); + } + + ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); + } + + TEST(JSONObject, objectWithPlaceholderAndObject) { + std::stringstream out; + { + JSONObject t(out); + auto l = t.placeholder("object"); + l.object().attr("key", "value"); + } + + ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); + } + + /* ---------------------------------------------------------------------------- + * JSONList + * --------------------------------------------------------------------------*/ + + TEST(JSONList, empty) { + std::stringstream out; + { + JSONList l(out); + } + ASSERT_EQ(out.str(), R"#([])#"); + } + + TEST(JSONList, withElements) { + std::stringstream out; + { + JSONList l(out); + l.elem("one"); + l.object(); + l.placeholder().write("three"); + } + ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); + } +} + diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk new file mode 100644 index 000000000..815e18560 --- /dev/null +++ b/src/libutil/tests/local.mk @@ -0,0 +1,15 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_DIR := $(d) + +libutil-tests_INSTALL_DIR := + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_CXXFLAGS += -I src/libutil -I src/libexpr + +libutil-tests_LIBS = libutil + +libutil-tests_LDFLAGS := $(GTEST_LIBS) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc new file mode 100644 index 000000000..4cb54995b --- /dev/null +++ b/src/libutil/tests/logging.cc @@ -0,0 +1,255 @@ +#include "logging.hh" +#include "nixexpr.hh" +#include "util.hh" + +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * logEI + * --------------------------------------------------------------------------*/ + + TEST(logEI, catpuresBasicProperties) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + throw TestError("an error for testing purposes"); + } catch (Error &e) { + testing::internal::CaptureStderr(); + logger->logEI(e.info()); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(),"\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\nan error for testing purposes\n"); + } + } + + TEST(logEI, appendingHintsToPreviousError) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + auto e = Error("initial error"); + throw TestError(e.info()); + } catch (Error &e) { + ErrorInfo ei = e.info(); + ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : "")); + + testing::internal::CaptureStderr(); + logger->logEI(ei); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- TestError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0minitial error\x1B[0m; subsequent error message.\n"); + } + + } + + TEST(logEI, picksUpSysErrorExitCode) { + + MakeError(TestError, Error); + ErrorInfo::programName = std::optional("error-unit-test"); + + try { + auto x = readFile(-1); + } + catch (SysError &e) { + testing::internal::CaptureStderr(); + logError(e.info()); + auto str = testing::internal::GetCapturedStderr(); + + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\n\x1B[33;1m\x1B[0mstatting file\x1B[0m: \x1B[33;1mBad file descriptor\x1B[0m\n"); + + } + } + + TEST(logEI, loggingErrorOnInfoLevel) { + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlInfo, + .name = "Info name", + .description = "Info description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1minfo:\x1B[0m\x1B[34;1m --- Info name --- error-unit-test\x1B[0m\nInfo description\n"); + } + + TEST(logEI, loggingErrorOnTalkativeLevel) { + verbosity = lvlTalkative; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlTalkative, + .name = "Talkative name", + .description = "Talkative description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mtalk:\x1B[0m\x1B[34;1m --- Talkative name --- error-unit-test\x1B[0m\nTalkative description\n"); + } + + TEST(logEI, loggingErrorOnChattyLevel) { + verbosity = lvlChatty; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlChatty, + .name = "Chatty name", + .description = "Talkative description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mchat:\x1B[0m\x1B[34;1m --- Chatty name --- error-unit-test\x1B[0m\nTalkative description\n"); + } + + TEST(logEI, loggingErrorOnDebugLevel) { + verbosity = lvlDebug; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlDebug, + .name = "Debug name", + .description = "Debug description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mdebug:\x1B[0m\x1B[34;1m --- Debug name --- error-unit-test\x1B[0m\nDebug description\n"); + } + + TEST(logEI, loggingErrorOnVomitLevel) { + verbosity = lvlVomit; + + testing::internal::CaptureStderr(); + + logger->logEI({ .level = lvlVomit, + .name = "Vomit name", + .description = "Vomit description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[32;1mvomit:\x1B[0m\x1B[34;1m --- Vomit name --- error-unit-test\x1B[0m\nVomit description\n"); + } + + /* ---------------------------------------------------------------------------- + * logError + * --------------------------------------------------------------------------*/ + + + TEST(logError, logErrorWithoutHintOrCode) { + testing::internal::CaptureStderr(); + + logError({ + .name = "name", + .description = "error description", + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n"); + } + + TEST(logError, logErrorWithPreviousAndNextLinesOfCode) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .description = "error with code lines", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = "previous line of code", + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = "next line of code", + }}); + + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + + TEST(logError, logErrorWithoutLinesOfCode) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .description = "error without any code lines.", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + + TEST(logError, logErrorWithOnlyHintAndName) { + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + testing::internal::CaptureStderr(); + + logError({ + .name = "error name", + .hint = hintfmt("hint %1%", "only"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13) + }}); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n"); + + } + + /* ---------------------------------------------------------------------------- + * logWarning + * --------------------------------------------------------------------------*/ + + TEST(logWarning, logWarningWithNameDescriptionAndHint) { + testing::internal::CaptureStderr(); + + logWarning({ + .name = "name", + .description = "error description", + .hint = hintfmt("there was a %1%", "warning"), + }); + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n"); + } + + TEST(logWarning, logWarningWithFileLineNumAndCode) { + + SymbolTable testTable; + auto problem_file = testTable.create("myfile.nix"); + + testing::internal::CaptureStderr(); + + logWarning({ + .name = "warning name", + .description = "warning description", + .hint = hintfmt("this hint has %1% templated %2%!!", + "yellow", + "values"), + .nixCode = NixCode { + .errPos = Pos(problem_file, 40, 13), + .prevLineOfCode = std::nullopt, + .errLineOfCode = "this is the problem line of code", + .nextLineOfCode = std::nullopt + }}); + + + auto str = testing::internal::GetCapturedStderr(); + ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + } + +} diff --git a/src/libutil/tests/lru-cache.cc b/src/libutil/tests/lru-cache.cc new file mode 100644 index 000000000..091d3d5ed --- /dev/null +++ b/src/libutil/tests/lru-cache.cc @@ -0,0 +1,130 @@ +#include "lru-cache.hh" +#include <gtest/gtest.h> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * size + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, sizeOfEmptyCacheIsZero) { + LRUCache<std::string, std::string> c(10); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, sizeOfSingleElementCacheIsOne) { + LRUCache<std::string, std::string> c(10); + c.upsert("foo", "bar"); + ASSERT_EQ(c.size(), 1); + } + + /* ---------------------------------------------------------------------------- + * upsert / get + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, getFromEmptyCache) { + LRUCache<std::string, std::string> c(10); + auto val = c.get("x"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, getExistingValue) { + LRUCache<std::string, std::string> c(10); + c.upsert("foo", "bar"); + auto val = c.get("foo"); + ASSERT_EQ(val, "bar"); + } + + TEST(LRUCache, getNonExistingValueFromNonEmptyCache) { + LRUCache<std::string, std::string> c(10); + c.upsert("foo", "bar"); + auto val = c.get("another"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, upsertOnZeroCapacityCache) { + LRUCache<std::string, std::string> c(0); + c.upsert("foo", "bar"); + auto val = c.get("foo"); + ASSERT_EQ(val.has_value(), false); + } + + TEST(LRUCache, updateExistingValue) { + LRUCache<std::string, std::string> c(1); + c.upsert("foo", "bar"); + + auto val = c.get("foo"); + ASSERT_EQ(val.value_or("error"), "bar"); + ASSERT_EQ(c.size(), 1); + + c.upsert("foo", "changed"); + val = c.get("foo"); + ASSERT_EQ(val.value_or("error"), "changed"); + ASSERT_EQ(c.size(), 1); + } + + TEST(LRUCache, overwriteOldestWhenCapacityIsReached) { + LRUCache<std::string, std::string> c(3); + c.upsert("one", "eins"); + c.upsert("two", "zwei"); + c.upsert("three", "drei"); + + ASSERT_EQ(c.size(), 3); + ASSERT_EQ(c.get("one").value_or("error"), "eins"); + + // exceed capacity + c.upsert("another", "whatever"); + + ASSERT_EQ(c.size(), 3); + // Retrieving "one" makes it the most recent element thus + // two will be the oldest one and thus replaced. + ASSERT_EQ(c.get("two").has_value(), false); + ASSERT_EQ(c.get("another").value(), "whatever"); + } + + /* ---------------------------------------------------------------------------- + * clear + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, clearEmptyCache) { + LRUCache<std::string, std::string> c(10); + c.clear(); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, clearNonEmptyCache) { + LRUCache<std::string, std::string> c(10); + c.upsert("one", "eins"); + c.upsert("two", "zwei"); + c.upsert("three", "drei"); + ASSERT_EQ(c.size(), 3); + c.clear(); + ASSERT_EQ(c.size(), 0); + } + + /* ---------------------------------------------------------------------------- + * erase + * --------------------------------------------------------------------------*/ + + TEST(LRUCache, eraseFromEmptyCache) { + LRUCache<std::string, std::string> c(10); + ASSERT_EQ(c.erase("foo"), false); + ASSERT_EQ(c.size(), 0); + } + + TEST(LRUCache, eraseMissingFromNonEmptyCache) { + LRUCache<std::string, std::string> c(10); + c.upsert("one", "eins"); + ASSERT_EQ(c.erase("foo"), false); + ASSERT_EQ(c.size(), 1); + ASSERT_EQ(c.get("one").value_or("error"), "eins"); + } + + TEST(LRUCache, eraseFromNonEmptyCache) { + LRUCache<std::string, std::string> c(10); + c.upsert("one", "eins"); + ASSERT_EQ(c.erase("one"), true); + ASSERT_EQ(c.size(), 0); + ASSERT_EQ(c.get("one").value_or("empty"), "empty"); + } +} diff --git a/src/libutil/tests/pool.cc b/src/libutil/tests/pool.cc new file mode 100644 index 000000000..127e42dda --- /dev/null +++ b/src/libutil/tests/pool.cc @@ -0,0 +1,127 @@ +#include "pool.hh" +#include <gtest/gtest.h> + +namespace nix { + + struct TestResource + { + + TestResource() { + static int counter = 0; + num = counter++; + } + + int dummyValue = 1; + bool good = true; + int num; + }; + + /* ---------------------------------------------------------------------------- + * Pool + * --------------------------------------------------------------------------*/ + + TEST(Pool, freshPoolHasZeroCountAndSpecifiedCapacity) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + ASSERT_EQ(pool.count(), 0); + ASSERT_EQ(pool.capacity(), 1); + } + + TEST(Pool, freshPoolCanGetAResource) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + ASSERT_EQ(pool.count(), 0); + + TestResource r = *(pool.get()); + + ASSERT_EQ(pool.count(), 1); + ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(r.dummyValue, 1); + ASSERT_EQ(r.good, true); + } + + TEST(Pool, capacityCanBeIncremented) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.incCapacity(); + ASSERT_EQ(pool.capacity(), 2); + } + + TEST(Pool, capacityCanBeDecremented) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.decCapacity(); + ASSERT_EQ(pool.capacity(), 0); + } + + TEST(Pool, flushBadDropsOutOfScopeResources) { + auto isGood = [](const ref<TestResource> & r) { return false; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + { + auto _r = pool.get(); + ASSERT_EQ(pool.count(), 1); + } + + pool.flushBad(); + ASSERT_EQ(pool.count(), 0); + } + + // Test that the resources we allocate are being reused when they are still good. + TEST(Pool, reuseResource) { + auto isGood = [](const ref<TestResource> & r) { return true; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them to be equal + // as the pool should hand out the same (still) good one again. + int counter = -1; + { + Pool<TestResource>::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { // the second handle should contain the same resource (with the same counter value) + Pool<TestResource>::Handle h = pool.get(); + ASSERT_EQ(h->num, counter); + } + } + + // Test that the resources we allocate are being thrown away when they are no longer good. + TEST(Pool, badResourceIsNotReused) { + auto isGood = [](const ref<TestResource> & r) { return false; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them + // to *not* be equal as the pool should hand out a new instance after + // the first one was returned. + int counter = -1; + { + Pool<TestResource>::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { + // the second handle should contain a different resource (with a + //different counter value) + Pool<TestResource>::Handle h = pool.get(); + ASSERT_NE(h->num, counter); + } + } +} diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc new file mode 100644 index 000000000..8e77ccbe1 --- /dev/null +++ b/src/libutil/tests/tests.cc @@ -0,0 +1,589 @@ +#include "util.hh" +#include "types.hh" + +#include <gtest/gtest.h> + +namespace nix { + +/* ----------- tests for util.hh ------------------------------------------------*/ + + /* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + + TEST(absPath, doesntChangeRoot) { + auto p = absPath("/"); + + ASSERT_EQ(p, "/"); + } + + + + + TEST(absPath, turnsEmptyPathIntoCWD) { + char cwd[PATH_MAX+1]; + auto p = absPath(""); + + ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); + } + + TEST(absPath, usesOptionalBasePathWhenGiven) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_EQ(p, cwd); + } + + TEST(absPath, isIdempotent) { + char _cwd[PATH_MAX+1]; + char* cwd = getcwd((char*)&_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); + } + + + TEST(absPath, pathIsCanonicalised) { + auto path = "/some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, "/some/path/with/trailing/dot"); + ASSERT_EQ(p1, p2); + } + + /* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + + TEST(canonPath, removesTrailingSlashes) { + auto path = "/this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_EQ(p, "/this/is/a/path"); + } + + TEST(canonPath, removesDots) { + auto path = "/this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_EQ(p, "/this/is/a/path"); + } + + TEST(canonPath, removesDots2) { + auto path = "/this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_EQ(p, "/this/is/a/path"); + } + + TEST(canonPath, requiresAbsolutePath) { + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); + } + + /* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + + TEST(dirOf, returnsEmptyStringForRoot) { + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); + } + + TEST(dirOf, returnsFirstPathComponent) { + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); + } + + /* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + + TEST(baseNameOf, emptyPath) { + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); + } + + TEST(baseNameOf, pathOnRoot) { + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); + } + + TEST(baseNameOf, relativePath) { + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); + } + + TEST(baseNameOf, pathWithTrailingSlashRoot) { + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); + } + + TEST(baseNameOf, trailingSlash) { + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); + } + + /* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + + TEST(isInDir, trivialCase) { + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); + } + + TEST(isInDir, notInDir) { + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); + } + + // XXX: hm, bug or feature? :) Looking at the implementation + // this might be problematic. + TEST(isInDir, emptyDir) { + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); + } + + /* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + + TEST(isDirOrInDir, trueForSameDirectory) { + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); + } + + TEST(isDirOrInDir, trueForEmptyPaths) { + ASSERT_EQ(isDirOrInDir("", ""), true); + } + + TEST(isDirOrInDir, falseForDisjunctPaths) { + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); + } + + TEST(isDirOrInDir, relativePaths) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); + } + + // XXX: while it is possible to use "." or ".." in the + // first argument this doesn't seem to work in the second. + TEST(isDirOrInDir, DISABLED_shouldWork) { + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); + + } + + /* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + + TEST(pathExists, rootExists) { + ASSERT_TRUE(pathExists("/")); + } + + TEST(pathExists, cwdExists) { + ASSERT_TRUE(pathExists(".")); + } + + TEST(pathExists, bogusPathDoesNotExist) { + ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + } + + /* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + + TEST(concatStringsSep, buildCommaSeparatedString) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); + } + + TEST(concatStringsSep, buildStringWithEmptySeparator) { + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); + } + + TEST(concatStringsSep, buildSingleString) { + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); + } + + /* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + + TEST(hasPrefix, emptyStringHasNoPrefix) { + ASSERT_FALSE(hasPrefix("", "foo")); + } + + TEST(hasPrefix, emptyStringIsAlwaysPrefix) { + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); + } + + TEST(hasPrefix, trivialCase) { + ASSERT_TRUE(hasPrefix("foobar", "foo")); + } + + /* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + + TEST(hasSuffix, emptyStringHasNoSuffix) { + ASSERT_FALSE(hasSuffix("", "foo")); + } + + TEST(hasSuffix, trivialCase) { + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); + } + + /* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + + TEST(base64Encode, emptyString) { + ASSERT_EQ(base64Encode(""), ""); + } + + TEST(base64Encode, encodesAString) { + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); + } + + TEST(base64Encode, encodeAndDecode) { + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); + } + + /* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + + TEST(base64Decode, emptyString) { + ASSERT_EQ(base64Decode(""), ""); + } + + TEST(base64Decode, decodeAString) { + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); + } + + /* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + + TEST(toLower, emptyString) { + ASSERT_EQ(toLower(""), ""); + } + + TEST(toLower, nonLetters) { + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); + } + + // std::tolower() doesn't handle unicode characters. In the context of + // store paths this isn't relevant but doesn't hurt to record this behavior + // here. + TEST(toLower, umlauts) { + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); + } + + /* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + + TEST(string2Float, emptyString) { + double n; + ASSERT_EQ(string2Float("", n), false); + } + + TEST(string2Float, trivialConversions) { + double n; + ASSERT_EQ(string2Float("1.0", n), true); + ASSERT_EQ(n, 1.0); + + ASSERT_EQ(string2Float("0.0", n), true); + ASSERT_EQ(n, 0.0); + + ASSERT_EQ(string2Float("-100.25", n), true); + ASSERT_EQ(n, (-100.25)); + } + + /* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + + TEST(string2Int, emptyString) { + double n; + ASSERT_EQ(string2Int("", n), false); + } + + TEST(string2Int, trivialConversions) { + double n; + ASSERT_EQ(string2Int("1", n), true); + ASSERT_EQ(n, 1); + + ASSERT_EQ(string2Int("0", n), true); + ASSERT_EQ(n, 0); + + ASSERT_EQ(string2Int("-100", n), true); + ASSERT_EQ(n, (-100)); + } + + /* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + + TEST(statusOk, zeroIsOk) { + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); + } + + + /* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + + TEST(rewriteStrings, emptyString) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); + } + + TEST(rewriteStrings, emptyRewrites) { + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); + } + + TEST(rewriteStrings, successfulRewrite) { + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); + } + + TEST(rewriteStrings, doesntOccur) { + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + + TEST(replaceStrings, emptyString) { + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); + } + + TEST(replaceStrings, successfulReplace) { + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); + } + + TEST(replaceStrings, doesntOccur) { + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); + } + + /* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + + TEST(trim, emptyString) { + ASSERT_EQ(trim(""), ""); + } + + TEST(trim, removesWhitespace) { + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + + TEST(chomp, emptyString) { + ASSERT_EQ(chomp(""), ""); + } + + TEST(chomp, removesWhitespace) { + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); + } + + /* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + + TEST(quoteStrings, empty) { + Strings s = { }; + Strings expected = { }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, emptyStrings) { + Strings s = { "", "", "" }; + Strings expected = { "''", "''", "''" }; + ASSERT_EQ(quoteStrings(s), expected); + + } + + TEST(quoteStrings, trivialQuote) { + Strings s = { "foo", "bar", "baz" }; + Strings expected = { "'foo'", "'bar'", "'baz'" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + TEST(quoteStrings, quotedStrings) { + Strings s = { "'foo'", "'bar'", "'baz'" }; + Strings expected = { "''foo''", "''bar''", "''baz''" }; + + ASSERT_EQ(quoteStrings(s), expected); + } + + /* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + + TEST(tokenizeString, empty) { + Strings expected = { }; + + ASSERT_EQ(tokenizeString<Strings>(""), expected); + } + + TEST(tokenizeString, tokenizeSpacesWithDefaults) { + auto s = "foo bar baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString<Strings>(s), expected); + } + + TEST(tokenizeString, tokenizeTabsWithDefaults) { + auto s = "foo\tbar\tbaz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString<Strings>(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { + auto s = "foo\t bar\t baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString<Strings>(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { + auto s = "foo\t\n bar\t\n baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString<Strings>(s), expected); + } + + TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString<Strings>(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = { "foo", "bar", "baz" }; + + ASSERT_EQ(tokenizeString<Strings>(s2), expected2); + } + + TEST(tokenizeString, tokenizeWithCustomSep) { + auto s = "foo\n,bar\n,baz\n"; + Strings expected = { "foo\n", "bar\n", "baz\n" }; + + ASSERT_EQ(tokenizeString<Strings>(s, ","), expected); + } + + /* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + + TEST(get, emptyContainer) { + StringMap s = { }; + auto expected = std::nullopt; + + ASSERT_EQ(get(s, "one"), expected); + } + + TEST(get, getFromContainer) { + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(get(s, "one"), expected); + } + + /* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + + TEST(filterANSIEscapes, emptyString) { + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); + } + + TEST(filterANSIEscapes, doesntChangePrintableChars) { + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); + } + + TEST(filterANSIEscapes, filtersColorCodes) { + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); + } + + TEST(filterANSIEscapes, expandsTabs) { + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); + } +} diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc new file mode 100644 index 000000000..80646ad3e --- /dev/null +++ b/src/libutil/tests/url.cc @@ -0,0 +1,266 @@ +#include "url.hh" +#include <gtest/gtest.h> + +namespace nix { + +/* ----------- tests for url.hh --------------------------------------------------*/ + + string print_map(std::map<string, string> m) { + std::map<string, string>::iterator it; + string s = "{ "; + for (it = m.begin(); it != m.end(); ++it) { + s += "{ "; + s += it->first; + s += " = "; + s += it->second; + s += " } "; + } + s += "}"; + return s; + } + + + std::ostream& operator<<(std::ostream& os, const ParsedURL& p) { + return os << "\n" + << "url: " << p.url << "\n" + << "base: " << p.base << "\n" + << "scheme: " << p.scheme << "\n" + << "authority: " << p.authority.value() << "\n" + << "path: " << p.path << "\n" + << "query: " << print_map(p.query) << "\n" + << "fragment: " << p.fragment << "\n"; + } + + TEST(parseURL, parsesSimpleHttpUrl) { + auto s = "http://www.example.org/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://www.example.org/file.tar.gz", + .base = "http://www.example.org/file.tar.gz", + .scheme = "http", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpsUrl) { + auto s = "https://www.example.org/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "https://www.example.org/file.tar.gz", + .base = "https://www.example.org/file.tar.gz", + .scheme = "https", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) { + auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "https://www.example.org/file.tar.gz", + .base = "https://www.example.org/file.tar.gz", + .scheme = "https", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, + .fragment = "hello", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) { + auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://www.example.org/file.tar.gz", + .base = "http://www.example.org/file.tar.gz", + .scheme = "http", + .authority = "www.example.org", + .path = "/file.tar.gz", + .query = (StringMap) { { "field", "value" } }, + .fragment = "?foo=bar#", + }; + + ASSERT_EQ(parsed, expected); + } + + + TEST(parseURL, parseIPv4Address) { + auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://127.0.0.1:8080/file.tar.gz", + .base = "https://127.0.0.1:8080/file.tar.gz", + .scheme = "http", + .authority = "127.0.0.1:8080", + .path = "/file.tar.gz", + .query = (StringMap) { { "download", "fast" }, { "when", "now" } }, + .fragment = "hello", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parseIPv6Address) { + auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .scheme = "http", + .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .path = "", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + + TEST(parseURL, parseEmptyQueryParams) { + auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&"; + auto parsed = parseURL(s); + ASSERT_EQ(parsed.query, (StringMap) { }); + } + + TEST(parseURL, parseUserPassword) { + auto s = "http://user:pass@www.example.org:8080/file.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "http://user:pass@www.example.org/file.tar.gz", + .base = "http://user:pass@www.example.org/file.tar.gz", + .scheme = "http", + .authority = "user:pass@www.example.org:8080", + .path = "/file.tar.gz", + .query = (StringMap) { }, + .fragment = "", + }; + + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parseFileURLWithQueryAndFragment) { + auto s = "file:///none/of/your/business"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "", + .base = "", + .scheme = "file", + .authority = "", + .path = "/none/of/your/business", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + + } + + TEST(parseURL, parsedUrlsIsEqualToItself) { + auto s = "http://www.example.org/file.tar.gz"; + auto url = parseURL(s); + + ASSERT_TRUE(url == url); + } + + TEST(parseURL, parseFTPUrl) { + auto s = "ftp://ftp.nixos.org/downloads/nixos.iso"; + auto parsed = parseURL(s); + + ParsedURL expected { + .url = "ftp://ftp.nixos.org/downloads/nixos.iso", + .base = "ftp://ftp.nixos.org/downloads/nixos.iso", + .scheme = "ftp", + .authority = "ftp.nixos.org", + .path = "/downloads/nixos.iso", + .query = (StringMap) { }, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + } + + TEST(parseURL, parsesAnythingInUriFormat) { + auto s = "whatever://github.com/NixOS/nixpkgs.git"; + auto parsed = parseURL(s); + } + + TEST(parseURL, parsesAnythingInUriFormatWithoutDoubleSlash) { + auto s = "whatever:github.com/NixOS/nixpkgs.git"; + auto parsed = parseURL(s); + } + + TEST(parseURL, emptyStringIsInvalidURL) { + ASSERT_THROW(parseURL(""), Error); + } + + /* ---------------------------------------------------------------------------- + * decodeQuery + * --------------------------------------------------------------------------*/ + + TEST(decodeQuery, emptyStringYieldsEmptyMap) { + auto d = decodeQuery(""); + ASSERT_EQ(d, (StringMap) { }); + } + + TEST(decodeQuery, simpleDecode) { + auto d = decodeQuery("yi=one&er=two"); + ASSERT_EQ(d, ((StringMap) { { "yi", "one" }, { "er", "two" } })); + } + + TEST(decodeQuery, decodeUrlEncodedArgs) { + auto d = decodeQuery("arg=%3D%3D%40%3D%3D"); + ASSERT_EQ(d, ((StringMap) { { "arg", "==@==" } })); + } + + TEST(decodeQuery, decodeArgWithEmptyValue) { + auto d = decodeQuery("arg="); + ASSERT_EQ(d, ((StringMap) { { "arg", ""} })); + } + + /* ---------------------------------------------------------------------------- + * percentDecode + * --------------------------------------------------------------------------*/ + + TEST(percentDecode, decodesUrlEncodedString) { + string s = "==@=="; + string d = percentDecode("%3D%3D%40%3D%3D"); + ASSERT_EQ(d, s); + } + + TEST(percentDecode, multipleDecodesAreIdempotent) { + string once = percentDecode("%3D%3D%40%3D%3D"); + string twice = percentDecode(once); + + ASSERT_EQ(once, twice); + } + + TEST(percentDecode, trailingPercent) { + string s = "==@==%"; + string d = percentDecode("%3D%3D%40%3D%3D%25"); + + ASSERT_EQ(d, s); + } + +} diff --git a/src/libutil/tests/xml-writer.cc b/src/libutil/tests/xml-writer.cc new file mode 100644 index 000000000..adcde25c9 --- /dev/null +++ b/src/libutil/tests/xml-writer.cc @@ -0,0 +1,105 @@ +#include "xml-writer.hh" +#include <gtest/gtest.h> +#include <sstream> + +namespace nix { + + /* ---------------------------------------------------------------------------- + * XMLWriter + * --------------------------------------------------------------------------*/ + + TEST(XMLWriter, emptyObject) { + std::stringstream out; + { + XMLWriter t(false, out); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n"); + } + + TEST(XMLWriter, objectWithEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.openElement("foobar"); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\"></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithEmptyAttrs) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = {}; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithAttrsEscaping) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "<key>", "<value>" } + }; + t.openElement("foobar", attrs); + } + + // XXX: While "<value>" is escaped, "<key>" isn't which I think is a bug. + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar <key>=\"<value>\"></foobar>"); + } + + TEST(XMLWriter, objectWithElementWithAttrsIndented) { + std::stringstream out; + { + XMLWriter t(true, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.openElement("foobar", attrs); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\">\n</foobar>\n"); + } + + TEST(XMLWriter, writeEmptyElement) { + std::stringstream out; + { + XMLWriter t(false, out); + t.writeEmptyElement("foobar"); + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar />"); + } + + TEST(XMLWriter, writeEmptyElementWithAttributes) { + std::stringstream out; + { + XMLWriter t(false, out); + XMLAttrs attrs = { + { "foo", "bar" } + }; + t.writeEmptyElement("foobar", attrs); + + } + + ASSERT_EQ(out.str(), "<?xml version='1.0' encoding='utf-8'?>\n<foobar foo=\"bar\" />"); + } + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 20b96a85c..3af485fa0 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,160 +1,34 @@ #pragma once - #include "ref.hh" -#include <string> #include <list> #include <set> -#include <memory> #include <map> - -#include <boost/format.hpp> - -/* Before 4.7, gcc's std::exception uses empty throw() specifiers for - * its (virtual) destructor and what() in c++11 mode, in violation of spec - */ -#ifdef __GNUC__ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) -#define EXCEPTION_NEEDS_THROW_SPEC -#endif -#endif - +#include <vector> namespace nix { - -/* Inherit some names from other namespaces for convenience. */ -using std::string; using std::list; using std::set; using std::vector; -using boost::format; - - -/* A variadic template that does nothing. Useful to call a function - for all variadic arguments but ignoring the result. */ -struct nop { template<typename... T> nop(T...) {} }; - - -struct FormatOrString -{ - string s; - FormatOrString(const string & s) : s(s) { }; - FormatOrString(const format & f) : s(f.str()) { }; - FormatOrString(const char * s) : s(s) { }; -}; - - -/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - equivalent to ‘boost::format(format) % a_0 % ... % - ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - takes place). */ - -inline void formatHelper(boost::format & f) -{ -} - -template<typename T, typename... Args> -inline void formatHelper(boost::format & f, const T & x, const Args & ... args) -{ - formatHelper(f % x, args...); -} - -inline std::string fmt(const std::string & s) -{ - return s; -} - -inline std::string fmt(const char * s) -{ - return s; -} - -inline std::string fmt(const FormatOrString & fs) -{ - return fs.s; -} - -template<typename... Args> -inline std::string fmt(const std::string & fs, const Args & ... args) -{ - boost::format f(fs); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - formatHelper(f, args...); - return f.str(); -} - - -/* BaseError should generally not be caught, as it has Interrupted as - a subclass. Catch Error instead. */ -class BaseError : public std::exception -{ -protected: - string prefix_; // used for location traces etc. - string err; -public: - unsigned int status = 1; // exit status - - template<typename... Args> - BaseError(unsigned int status, const Args & ... args) - : err(fmt(args...)) - , status(status) - { - } - - template<typename... Args> - BaseError(const Args & ... args) - : err(fmt(args...)) - { - } - -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return err.c_str(); } -#else - const char * what() const noexcept { return err.c_str(); } -#endif - - const string & msg() const { return err; } - const string & prefix() const { return prefix_; } - BaseError & addPrefix(const FormatOrString & fs); -}; - -#define MakeError(newClass, superClass) \ - class newClass : public superClass \ - { \ - public: \ - using superClass::superClass; \ - } - -MakeError(Error, BaseError); - -class SysError : public Error -{ -public: - int errNo; - - template<typename... Args> - SysError(const Args & ... args) - : Error(addErrno(fmt(args...))) - { } - -private: - - std::string addErrno(const std::string & s); -}; - +using std::string; typedef list<string> Strings; typedef set<string> StringSet; -typedef std::map<std::string, std::string> StringMap; - +typedef std::map<string, string> StringMap; /* Paths are just strings. */ + typedef string Path; typedef list<Path> Paths; typedef set<Path> PathSet; +/* Helper class to run code at startup. */ +template<typename T> +struct OnStartup +{ + OnStartup(T && t) { t(); } +}; } diff --git a/src/libutil/url.cc b/src/libutil/url.cc new file mode 100644 index 000000000..88c09eef9 --- /dev/null +++ b/src/libutil/url.cc @@ -0,0 +1,138 @@ +#include "url.hh" +#include "util.hh" + +namespace nix { + +std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); +std::regex revRegex(revRegexS, std::regex::ECMAScript); +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + +ParsedURL parseURL(const std::string & url) +{ + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + if (std::regex_match(url, match, uriRegex)) { + auto & base = match[1]; + std::string scheme = match[2]; + auto authority = match[3].matched + ? std::optional<std::string>(match[3]) : std::nullopt; + std::string path = match[4].matched ? match[4] : match[5]; + auto & query = match[6]; + auto & fragment = match[7]; + + auto isFile = scheme.find("file") != std::string::npos; + + if (authority && *authority != "" && isFile) + throw Error("file:// URL '%s' has unexpected authority '%s'", + url, *authority); + + if (isFile && path.empty()) + path = "/"; + + return ParsedURL{ + .url = url, + .base = base, + .scheme = scheme, + .authority = authority, + .path = path, + .query = decodeQuery(query), + .fragment = percentDecode(std::string(fragment)) + }; + } + + else + throw BadURL("'%s' is not a valid URL", url); +} + +std::string percentDecode(std::string_view in) +{ + std::string decoded; + for (size_t i = 0; i < in.size(); ) { + if (in[i] == '%') { + if (i + 2 >= in.size()) + throw BadURL("invalid URI parameter '%s'", in); + try { + decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw BadURL("invalid URI parameter '%s'", in); + } + } else + decoded += in[i++]; + } + return decoded; +} + +std::map<std::string, std::string> decodeQuery(const std::string & query) +{ + std::map<std::string, std::string> result; + + for (auto s : tokenizeString<Strings>(query, "&")) { + auto e = s.find('='); + if (e != std::string::npos) + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); + } + + return result; +} + +std::string percentEncode(std::string_view s) +{ + std::string res; + for (auto & c : s) + if ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || strchr("-._~!$&'()*+,;=:@", c)) + res += c; + else + res += fmt("%%%02x", (unsigned int) c); + return res; +} + +std::string encodeQuery(const std::map<std::string, std::string> & ss) +{ + std::string res; + bool first = true; + for (auto & [name, value] : ss) { + if (!first) res += '&'; + first = false; + res += percentEncode(name); + res += '='; + res += percentEncode(value); + } + return res; +} + +std::string ParsedURL::to_string() const +{ + return + scheme + + ":" + + (authority ? "//" + *authority : "") + + path + + (query.empty() ? "" : "?" + encodeQuery(query)) + + (fragment.empty() ? "" : "#" + percentEncode(fragment)); +} + +bool ParsedURL::operator ==(const ParsedURL & other) const +{ + return + scheme == other.scheme + && authority == other.authority + && path == other.path + && query == other.query + && fragment == other.fragment; +} + +} diff --git a/src/libutil/url.hh b/src/libutil/url.hh new file mode 100644 index 000000000..2ef88ef2a --- /dev/null +++ b/src/libutil/url.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "error.hh" + +#include <regex> + +namespace nix { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional<std::string> authority; + std::string path; + std::map<std::string, std::string> query; + std::string fragment; + + std::string to_string() const; + + bool operator ==(const ParsedURL & other) const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map<std::string, std::string> decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); + +// URI stuff. +const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; +const static std::string schemeRegex = "(?:[a-z+]+)"; +const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; +const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; +const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; +const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)"; +const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")"; +const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)"; +const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; +const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; +const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// Instead of defining what a good Git Ref is, we define what a bad Git Ref is +// This is because of the definition of a ref in refs.c in https://github.com/git/git +// See tests/fetchGitRefs.sh for the full definition +const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; +extern std::regex badGitRefRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 012f1d071..667dd2edb 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -40,24 +40,6 @@ extern char * * environ; namespace nix { - -const std::string nativeSystem = SYSTEM; - - -BaseError & BaseError::addPrefix(const FormatOrString & fs) -{ - prefix_ = fs.s + prefix_; - return *this; -} - - -std::string SysError::addErrno(const std::string & s) -{ - errNo = errno; - return s + ": " + strerror(errNo); -} - - std::optional<std::string> getEnv(const std::string & key) { char * value = getenv(key.c_str()); @@ -97,10 +79,10 @@ void replaceEnv(std::map<std::string, std::string> newEnv) } -Path absPath(Path path, Path dir) +Path absPath(Path path, std::optional<Path> dir) { if (path[0] != '/') { - if (dir == "") { + if (!dir) { #ifdef __GNU__ /* GNU (aka. GNU/Hurd) doesn't have any limitation on path lengths and doesn't define `PATH_MAX'. */ @@ -116,7 +98,7 @@ Path absPath(Path path, Path dir) free(buf); #endif } - path = dir + "/" + path; + path = *dir + "/" + path; } return canonPath(path); } @@ -129,7 +111,7 @@ Path canonPath(const Path & path, bool resolveSymlinks) string s; if (path[0] != '/') - throw Error(format("not an absolute path: '%1%'") % path); + throw Error("not an absolute path: '%1%'", path); string::const_iterator i = path.begin(), end = path.end(); string temp; @@ -165,7 +147,7 @@ Path canonPath(const Path & path, bool resolveSymlinks) the symlink target might contain new symlinks). */ if (resolveSymlinks && isLink(s)) { if (++followCount >= maxFollow) - throw Error(format("infinite symlink recursion in path '%1%'") % path); + throw Error("infinite symlink recursion in path '%1%'", path); temp = absPath(readLink(s), dirOf(s)) + string(i, end); i = temp.begin(); /* restart */ @@ -226,7 +208,7 @@ struct stat lstat(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); return st; } @@ -238,7 +220,7 @@ bool pathExists(const Path & path) res = lstat(path.c_str(), &st); if (!res) return true; if (errno != ENOENT && errno != ENOTDIR) - throw SysError(format("getting status of %1%") % path); + throw SysError("getting status of %1%", path); return false; } @@ -268,16 +250,13 @@ bool isLink(const Path & path) } -DirEntries readDirectory(const Path & path) +DirEntries readDirectory(DIR *dir, const Path & path) { DirEntries entries; entries.reserve(64); - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + while (errno = 0, dirent = readdir(dir)) { /* sic */ checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; @@ -289,11 +268,19 @@ DirEntries readDirectory(const Path & path) #endif ); } - if (errno) throw SysError(format("reading directory '%1%'") % path); + if (errno) throw SysError("reading directory '%1%'", path); return entries; } +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + unsigned char getFileType(const Path & path) { @@ -311,26 +298,24 @@ string readFile(int fd) if (fstat(fd, &st) == -1) throw SysError("statting file"); - std::vector<unsigned char> buf(st.st_size); - readFull(fd, buf.data(), st.st_size); - - return string((char *) buf.data(), st.st_size); + return drainFD(fd, true, st.st_size); } -string readFile(const Path & path, bool drain) +string readFile(const Path & path) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) - throw SysError(format("opening file '%1%'") % path); - return drain ? drainFD(fd.get()) : readFile(fd.get()); + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); } void readFile(const Path & path, Sink & sink) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%s'", path); + if (!fd) + throw SysError("opening file '%s'", path); drainFD(fd.get(), sink); } @@ -339,7 +324,7 @@ void writeFile(const Path & path, const string & s, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); writeFull(fd.get(), s); } @@ -348,7 +333,7 @@ void writeFile(const Path & path, Source & source, mode_t mode) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); std::vector<unsigned char> buf(64 * 1024); @@ -389,14 +374,16 @@ void writeLine(int fd, string s) } -static void _deletePath(const Path & path, unsigned long long & bytesFreed) +static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed) { checkInterrupt(); + string name(baseNameOf(path)); + struct stat st; - if (lstat(path.c_str(), &st) == -1) { + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) @@ -406,18 +393,43 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) /* Make the directory accessible. */ const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) - throw SysError(format("chmod '%1%'") % path); + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); } - for (auto & i : readDirectory(path)) - _deletePath(path + "/" + i.name, bytesFreed); + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (!fd) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); } - if (remove(path.c_str()) == -1) { + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + if (!dirfd) { + // This really shouldn't fail silently, but it's left this way + // for backwards compatibility. if (errno == ENOENT) return; - throw SysError(format("cannot unlink '%1%'") % path); + + throw SysError("opening directory '%1%'", path); } + + _deletePath(dirfd.get(), path, bytesFreed); } @@ -468,16 +480,27 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, "wheel", then "tar" will fail to unpack archives that have the setgid bit set on directories. */ if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError(format("setting group of directory '%1%'") % tmpDir); + throw SysError("setting group of directory '%1%'", tmpDir); #endif return tmpDir; } if (errno != EEXIST) - throw SysError(format("creating directory '%1%'") % tmpDir); + throw SysError("creating directory '%1%'", tmpDir); } } +std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + return {std::move(fd), tmpl}; +} + + std::string getUserName() { auto pw = getpwuid(geteuid()); @@ -544,15 +567,15 @@ Paths createDirs(const Path & path) if (lstat(path.c_str(), &st) == -1) { created = createDirs(dirOf(path)); if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError(format("creating directory '%1%'") % path); + throw SysError("creating directory '%1%'", path); st = lstat(path); created.push_back(path); } if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError(format("statting symlink '%1%'") % path); + throw SysError("statting symlink '%1%'", path); - if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path); + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); return created; } @@ -561,7 +584,7 @@ Paths createDirs(const Path & path) void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) - throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); + throw SysError("creating symlink from '%1%' to '%2%'", link, target); } @@ -578,7 +601,7 @@ void replaceSymlink(const Path & target, const Path & link) } if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); + throw SysError("renaming '%1%' to '%2%'", tmp, link); break; } @@ -622,9 +645,9 @@ void writeFull(int fd, const string & s, bool allowInterrupts) } -string drainFD(int fd, bool block) +string drainFD(int fd, bool block, const size_t reserveSize) { - StringSink sink; + StringSink sink(reserveSize); drainFD(fd, sink, block); return std::move(*sink.s); } @@ -683,7 +706,7 @@ AutoDelete::~AutoDelete() deletePath(path); else { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } } } catch (...) { @@ -749,7 +772,7 @@ void AutoCloseFD::close() if (fd != -1) { if (::close(fd) == -1) /* This should never happen. */ - throw SysError(format("closing file descriptor %1%") % fd); + throw SysError("closing file descriptor %1%", fd); } } @@ -822,7 +845,7 @@ int Pid::kill() { assert(pid != -1); - debug(format("killing process %1%") % pid); + debug("killing process %1%", pid); /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child @@ -834,7 +857,7 @@ int Pid::kill() #if __FreeBSD__ || __APPLE__ if (errno != EPERM || ::kill(pid, 0) != 0) #endif - printError((SysError("killing process %d", pid).msg())); + logError(SysError("killing process %d", pid).info()); } return wait(); @@ -880,7 +903,7 @@ pid_t Pid::release() void killUser(uid_t uid) { - debug(format("killing all processes running under uid '%1%'") % uid); + debug("killing all processes running under uid '%1%'", uid); assert(uid != 0); /* just to be safe... */ @@ -909,7 +932,7 @@ void killUser(uid_t uid) #endif if (errno == ESRCH) break; /* no more processes */ if (errno != EINTR) - throw SysError(format("cannot kill processes for uid '%1%'") % uid); + throw SysError("cannot kill processes for uid '%1%'", uid); } _exit(0); @@ -917,7 +940,7 @@ void killUser(uid_t uid) int status = pid.wait(); if (status != 0) - throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status)); + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); /* !!! We should really do some check to make sure that there are no processes left running under `uid', but there is no portable @@ -949,7 +972,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { auto wrapper = [&]() { if (!options.allowVfork) - logger = makeDefaultLogger(); + logger = makeSimpleLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -1205,28 +1228,6 @@ template StringSet tokenizeString(std::string_view s, const string & separators) template vector<string> tokenizeString(std::string_view s, const string & separators); -string concatStringsSep(const string & sep, const Strings & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - -string concatStringsSep(const string & sep, const StringSet & ss) -{ - string s; - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - - string chomp(const string & s) { size_t i = s.find_last_not_of(" \n\r\t"); @@ -1296,7 +1297,7 @@ bool statusOk(int status) } -bool hasPrefix(const string & s, const string & prefix) +bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; } @@ -1333,7 +1334,7 @@ void ignoreException() try { throw; } catch (std::exception & e) { - printError(format("error (ignored): %1%") % e.what()); + printError("error (ignored): %1%", e.what()); } } @@ -1390,7 +1391,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -string base64Encode(const string & s) +string base64Encode(std::string_view s) { string res; int data = 0, nbits = 0; @@ -1411,7 +1412,7 @@ string base64Encode(const string & s) } -string base64Decode(const string & s) +string base64Decode(std::string_view s) { bool init = false; char decode[256]; @@ -1446,17 +1447,6 @@ string base64Decode(const string & s) } -void callFailure(const std::function<void(std::exception_ptr exc)> & failure, std::exception_ptr exc) -{ - try { - failure(exc); - } catch (std::exception & e) { - printError(format("uncaught exception: %s") % e.what()); - abort(); - } -} - - static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3bfebcd15..3641daaec 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,7 +1,9 @@ #pragma once #include "types.hh" +#include "error.hh" #include "logging.hh" +#include "ansicolor.hh" #include <sys/types.h> #include <sys/stat.h> @@ -16,6 +18,7 @@ #include <sstream> #include <optional> #include <future> +#include <iterator> #ifndef HAVE_STRUCT_DIRENT_D_TYPE #define DT_UNKNOWN 0 @@ -46,7 +49,7 @@ void clearEnv(); /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, Path dir = ""); +Path absPath(Path path, std::optional<Path> dir = {}); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink @@ -56,12 +59,12 @@ Path canonPath(const Path & path, bool resolveSymlinks = false); /* Return the directory part of the given canonical path, i.e., everything before the final `/'. If the path is the root or an - immediate child thereof (e.g., `/foo'), this means an empty string - is returned. */ + immediate child thereof (e.g., `/foo'), this means `/' + is returned.*/ Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything - following the final `/'. */ + following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); /* Check whether 'path' is a descendant of 'dir'. */ @@ -101,7 +104,7 @@ unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); -string readFile(const Path & path, bool drain = false); +string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ @@ -122,10 +125,6 @@ void deletePath(const Path & path); void deletePath(const Path & path, unsigned long long & bytesFreed); -/* Create a temporary directory. */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - std::string getUserName(); /* Return $HOME or the user's home directory from /etc/passwd. */ @@ -164,7 +163,7 @@ MakeError(EndOfFile, Error); /* Read a file descriptor until EOF occurs. */ -string drainFD(int fd, bool block = true); +string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); @@ -205,6 +204,14 @@ public: }; +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a temporary file, returning a file handle and its path. */ +std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix"); + + class Pipe { public: @@ -345,8 +352,26 @@ template<class C> C tokenizeString(std::string_view s, const string & separators /* Concatenate the given strings with a separator between the elements. */ -string concatStringsSep(const string & sep, const Strings & ss); -string concatStringsSep(const string & sep, const StringSet & ss); +template<class C> +string concatStringsSep(const string & sep, const C & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +/* Add quotes around a collection of strings. */ +template<class C> Strings quoteStrings(const C & c) +{ + Strings res; + for (auto & s : c) + res.push_back("'" + s + "'"); + return res; +} /* Remove trailing whitespace from a string. */ @@ -365,17 +390,6 @@ string replaceStrings(const std::string & s, std::string rewriteStrings(const std::string & s, const StringMap & rewrites); -/* If a set contains 'from', remove it and insert 'to'. */ -template<typename T> -void replaceInSet(std::set<T> & set, const T & from, const T & to) -{ - auto i = set.find(from); - if (i == set.end()) return; - set.erase(i); - set.insert(to); -} - - /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); @@ -403,7 +417,7 @@ template<class N> bool string2Float(const string & s, N & n) /* Return true iff `s' starts with `prefix'. */ -bool hasPrefix(const string & s, const string & prefix); +bool hasPrefix(std::string_view s, std::string_view prefix); /* Return true iff `s' ends in `suffix'. */ @@ -423,14 +437,12 @@ std::string shellEscape(const std::string & s); void ignoreException(); -/* Some ANSI escape sequences. */ -#define ANSI_NORMAL "\e[0m" -#define ANSI_BOLD "\e[1m" -#define ANSI_FAINT "\e[2m" -#define ANSI_RED "\e[31;1m" -#define ANSI_GREEN "\e[32;1m" -#define ANSI_YELLOW "\e[33;1m" -#define ANSI_BLUE "\e[34;1m" + +/* Tree formatting. */ +constexpr char treeConn[] = "├───"; +constexpr char treeLast[] = "└───"; +constexpr char treeLine[] = "│ "; +constexpr char treeNull[] = " "; /* Truncate a string to 'width' printable characters. If 'filterAll' @@ -444,17 +456,17 @@ std::string filterANSIEscapes(const std::string & s, /* Base64 encoding/decoding. */ -string base64Encode(const string & s); -string base64Decode(const string & s); +string base64Encode(std::string_view s); +string base64Decode(std::string_view s); -/* Get a value for the specified key from an associate container, or a - default value if the key doesn't exist. */ +/* Get a value for the specified key from an associate container. */ template <class T> -std::optional<std::string> get(const T & map, const std::string & key) +std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - return i == map.end() ? std::optional<std::string>() : i->second; + if (i == map.end()) return {}; + return std::optional<typename T::mapped_type>(i->second); } @@ -558,4 +570,31 @@ extern PathFilter defaultPathFilter; AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +// A Rust/Python-like enumerate() iterator adapter. +// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. +template <typename T, + typename TIter = decltype(std::begin(std::declval<T>())), + typename = decltype(std::end(std::declval<T>()))> +constexpr auto enumerate(T && iterable) +{ + struct iterator + { + size_t i; + TIter iter; + bool operator != (const iterator & other) const { return iter != other.iter; } + void operator ++ () { ++i; ++iter; } + auto operator * () const { return std::tie(i, *iter); } + }; + + struct iterable_wrapper + { + T iterable; + auto begin() { return iterator{ 0, std::begin(iterable) }; } + auto end() { return iterator{ 0, std::end(iterable) }; } + }; + + return iterable_wrapper{ std::forward<T>(iterable) }; +} + + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 27ec7d0fe..a224d635d 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -16,7 +16,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; using namespace std::string_literals; @@ -314,7 +314,7 @@ static void _main(int argc, char * * argv) state->eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, i, *autoArgs, vRoot).first); state->forceValue(v); getDerivations(*state, v, "", *autoArgs, drvs, false); } @@ -363,12 +363,16 @@ static void _main(int argc, char * * argv) if (!drv) throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation"); - pathsToBuild.emplace_back(store->parseStorePath(drv->queryDrvPath())); + pathsToBuild.push_back({store->parseStorePath(drv->queryDrvPath())}); shell = drv->queryOutPath() + "/bin/bash"; } catch (Error & e) { - printError("warning: %s; will use bash from your environment", e.what()); + logWarning({ + .name = "bashInteractive", + .hint = hintfmt("%s; will use bash from your environment", + (e.info().hint ? e.info().hint->str() : "")) + }); shell = "bash"; } } @@ -377,9 +381,9 @@ static void _main(int argc, char * * argv) for (const auto & input : drv.inputDrvs) if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); })) - pathsToBuild.emplace_back(input.first, input.second); + pathsToBuild.push_back({input.first, input.second}); for (const auto & src : drv.inputSrcs) - pathsToBuild.emplace_back(src); + pathsToBuild.push_back({src}); buildPaths(pathsToBuild); @@ -472,6 +476,8 @@ static void _main(int argc, char * * argv) restoreSignals(); + logger->stop(); + execvp(shell->c_str(), argPtrs.data()); throw SysError("executing shell '%s'", *shell); @@ -493,7 +499,7 @@ static void _main(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); - pathsToBuild.emplace_back(store->parseStorePath(drvPath), StringSet{outputName}); + pathsToBuild.push_back({store->parseStorePath(drvPath), {outputName}}); std::string drvPrefix; auto i = drvPrefixes.find(drvPath); @@ -521,6 +527,8 @@ static void _main(int argc, char * * argv) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); + logger->stop(); + for (auto & path : outPaths) std::cout << path << '\n'; } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 1b337a712..3ccf620c9 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,8 +1,9 @@ #include "shared.hh" #include "globals.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" +#include "fetchers.hh" #include <fcntl.h> #include <regex> @@ -37,7 +38,7 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw SysError(format("opening '%1%' for writing") % channelsList); + throw SysError("opening '%1%' for writing", channelsList); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -46,9 +47,9 @@ static void writeChannels() static void addChannel(const string & url, const string & name) { if (!regex_search(url, std::regex("^(file|http|https)://"))) - throw Error(format("invalid channel URL '%1%'") % url); + throw Error("invalid channel URL '%1%'", url); if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) - throw Error(format("invalid channel identifier '%1%'") % name); + throw Error("invalid channel identifier '%1%'", name); readChannels(); channels[name] = url; writeChannels(); @@ -86,12 +87,9 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - CachedDownloadRequest request(url); - request.ttl = 0; - auto dl = getDownloader(); - auto result = dl->downloadCached(store, request); - auto filename = result.path; - url = chomp(result.effectiveUri); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto filename = store->toRealPath(result.storePath); + url = result.effectiveUrl; // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -114,11 +112,10 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; - } catch (DownloadError & e) { - filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + } catch (FileTransferError & e) { + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); } - chomp(filename); } // Regardless of where it came from, add the expression representing this channel to accumulated expression @@ -140,9 +137,9 @@ static void update(const StringSet & channelNames) if (S_ISLNK(st.st_mode)) // old-skool ~/.nix-defexpr if (unlink(nixDefExpr.c_str()) == -1) - throw SysError(format("unlinking %1%") % nixDefExpr); + throw SysError("unlinking %1%", nixDefExpr); } else if (errno != ENOENT) { - throw SysError(format("getting status of %1%") % nixDefExpr); + throw SysError("getting status of %1%", nixDefExpr); } createDirs(nixDefExpr); auto channelLink = nixDefExpr + "/channels"; @@ -185,6 +182,8 @@ static int _main(int argc, char ** argv) } else if (*arg == "--rollback") { cmd = cRollback; } else { + if (hasPrefix(*arg, "-")) + throw UsageError("unsupported argument '%s'", *arg); args.push_back(std::move(*arg)); } return true; diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index d4060ac93..aa5ada3a6 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,7 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include <iostream> #include <cerrno> diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index f87035760..b10184718 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "store-api.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" using namespace nix; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 134898561..bcb86cbce 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "derivations.hh" #include "finally.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include "daemon.hh" #include <algorithm> @@ -36,7 +36,7 @@ using namespace nix::daemon; #define SPLICE_F_MOVE 0 static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) { - /* We ignore most parameters, we just have them for conformance with the linux syscall */ + // We ignore most parameters, we just have them for conformance with the linux syscall std::vector<char> buf(8192); auto read_count = read(fd_in, buf.data(), buf.size()); if (read_count == -1) @@ -57,7 +57,7 @@ static void sigChldHandler(int sigNo) { // Ensure we don't modify errno of whatever we've interrupted auto saved_errno = errno; - /* Reap all dead children. */ + // Reap all dead children. while (waitpid(-1, 0, WNOHANG) > 0) ; errno = saved_errno; } @@ -106,7 +106,7 @@ struct PeerInfo }; -/* Get the identity of the caller, if possible. */ +// Get the identity of the caller, if possible. static PeerInfo getPeerInfo(int remote) { PeerInfo peer = { false, 0, false, 0, false, 0 }; @@ -154,13 +154,12 @@ static void daemonLoop(char * * argv) if (chdir("/") == -1) throw SysError("cannot change current directory"); - /* Get rid of children automatically; don't let them become - zombies. */ + // Get rid of children automatically; don't let them become zombies. setSigChldAction(true); AutoCloseFD fdSocket; - /* Handle socket-based activation by systemd. */ + // Handle socket-based activation by systemd. auto listenFds = getEnv("LISTEN_FDS"); if (listenFds) { if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") @@ -169,17 +168,17 @@ static void daemonLoop(char * * argv) closeOnExec(fdSocket.get()); } - /* Otherwise, create and bind to a Unix domain socket. */ + // Otherwise, create and bind to a Unix domain socket. else { createDirs(dirOf(settings.nixDaemonSocketFile)); fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } - /* Loop accepting connections. */ + // Loop accepting connections. while (1) { try { - /* Accept a connection. */ + // Accept a connection. struct sockaddr_un remoteAddr; socklen_t remoteAddrLen = sizeof(remoteAddr); @@ -209,13 +208,13 @@ static void daemonLoop(char * * argv) trusted = Trusted; if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) - throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") % (peer.uidKnown ? user : "<unknown>")); - /* Fork a child to handle the connection. */ + // Fork a child to handle the connection. ProcessOptions options; options.errorPrefix = "unexpected Nix daemon error: "; options.dieWithParent = false; @@ -224,20 +223,20 @@ static void daemonLoop(char * * argv) startProcess([&]() { fdSocket = -1; - /* Background the daemon. */ + // Background the daemon. if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); - /* Restore normal handling of SIGCHLD. */ + // Restore normal handling of SIGCHLD. setSigChldAction(false); - /* For debugging, stuff the pid into argv[1]. */ + // For debugging, stuff the pid into argv[1]. if (peer.pidKnown && argv[1]) { string processName = std::to_string(peer.pid); strncpy(argv[1], processName.c_str(), strlen(argv[1])); } - /* Handle the connection. */ + // Handle the connection. FdSource from(remote.get()); FdSink to(remote.get()); processConnection(openUncachedStore(), from, to, trusted, NotRecursive, user, peer.uid); @@ -247,8 +246,11 @@ static void daemonLoop(char * * argv) } catch (Interrupted & e) { return; - } catch (Error & e) { - printError(format("error processing connection: %1%") % e.msg()); + } catch (Error & error) { + ErrorInfo ei = error.info(); + ei.hint = std::optional(hintfmt("error processing connection: %1%", + (error.info().hint.has_value() ? error.info().hint->str() : ""))); + logError(ei); } } } @@ -261,7 +263,7 @@ static int _main(int argc, char * * argv) parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") - ; /* ignored for backwards compatibility */ + ; // ignored for backwards compatibility else if (*arg == "--help") showManPage("nix-daemon"); else if (*arg == "--version") @@ -276,7 +278,7 @@ static int _main(int argc, char * * argv) if (stdio) { if (getStoreType() == tDaemon) { - /* Forward on this connection to the real daemon */ + // Forward on this connection to the real daemon auto socketPath = settings.nixDaemonSocketFile; auto s = socket(PF_UNIX, SOCK_STREAM, 0); if (s == -1) @@ -284,17 +286,17 @@ static int _main(int argc, char * * argv) auto socketDir = dirOf(socketPath); if (chdir(socketDir.c_str()) == -1) - throw SysError(format("changing to socket directory '%1%'") % socketDir); + throw SysError("changing to socket directory '%1%'", socketDir); auto socketName = std::string(baseNameOf(socketPath)); auto addr = sockaddr_un{}; addr.sun_family = AF_UNIX; if (socketName.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket name %1% is too long") % socketName); + throw Error("socket name %1% is too long", socketName); strcpy(addr.sun_path, socketName.c_str()); if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at %1%") % socketPath); + throw SysError("cannot connect to daemon at %1%", socketPath); auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; while (true) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 106dfe0b6..8b0692035 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -13,7 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include <cerrno> #include <ctime> @@ -25,7 +25,6 @@ #include <sys/stat.h> #include <unistd.h> - using namespace nix; using std::cout; @@ -70,8 +69,7 @@ typedef void (* Operation) (Globals & globals, static string needArg(Strings::iterator & i, Strings & args, const string & arg) { - if (i == args.end()) throw UsageError( - format("'%1%' requires an argument") % arg); + if (i == args.end()) throw UsageError("'%1%' requires an argument", arg); return *i++; } @@ -125,7 +123,10 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); if (!attrs.insert(attrName).second) { - printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); + logError({ + .name = "Name collision", + .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2) + }); continue; } /* Load the expression on demand. */ @@ -133,7 +134,7 @@ static void getAllExprs(EvalState & state, Value & vArg(*state.allocValue()); mkString(vArg, path2); if (v.attrs->size() == v.attrs->capacity()) - throw Error(format("too many Nix expressions in directory '%1%'") % path); + throw Error("too many Nix expressions in directory '%1%'", path); mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); } else if (S_ISDIR(st.st_mode)) @@ -144,11 +145,12 @@ static void getAllExprs(EvalState & state, } + static void loadSourceExpr(EvalState & state, const Path & path, Value & v) { struct stat st; if (stat(path.c_str(), &st) == -1) - throw SysError(format("getting information about '%1%'") % path); + throw SysError("getting information about '%1%'", path); if (isNixExpr(path, st)) state.evalFile(path, v); @@ -178,7 +180,7 @@ static void loadDerivations(EvalState & state, Path nixExprPath, Value vRoot; loadSourceExpr(state, nixExprPath, vRoot); - Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot).first); getDerivations(state, v, pathPrefix, autoArgs, elems, true); @@ -210,9 +212,7 @@ static bool isPrebuilt(EvalState & state, DrvInfo & elem) { auto path = state.store->parseStorePath(elem.queryOutPath()); if (state.store->isValidPath(path)) return true; - StorePathSet paths; - paths.insert(path.clone()); // FIXME: why doesn't StorePathSet{path.clone()} work? - return state.store->querySubstitutablePaths(paths).count(path); + return state.store->querySubstitutablePaths({path}).count(path); } @@ -221,7 +221,7 @@ static void checkSelectorUse(DrvNames & selectors) /* Check that all selectors have been used. */ for (auto & i : selectors) if (i.hits == 0 && i.fullName != "*") - throw Error(format("selector '%1%' matches no derivations") % i.fullName); + throw Error("selector '%1%' matches no derivations", i.fullName); } @@ -408,7 +408,7 @@ static void queryInstSources(EvalState & state, Value vRoot; loadSourceExpr(state, instSource.nixExprPath, vRoot); for (auto & i : args) { - Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot).first); getDerivations(state, v, "", *instSource.autoArgs, elems, true); } break; @@ -423,9 +423,9 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.emplace_back(state.store->parseStorePath(drvPath)); + targets.push_back({state.store->parseStorePath(drvPath)}); else - targets.emplace_back(state.store->parseStorePath(i.queryOutPath())); + targets.push_back({state.store->parseStorePath(i.queryOutPath())}); } printMissing(state.store, targets); @@ -507,7 +507,7 @@ static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) globals.preserveInstalled = true; else if (arg == "--remove-all" || arg == "-r") globals.removeAll = true; - else throw UsageError(format("unknown flag '%1%'") % arg); + else throw UsageError("unknown flag '%1%'", arg); } installDerivations(globals, opArgs, globals.profile); @@ -618,7 +618,7 @@ static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--leq") upgradeType = utLeq; else if (arg == "--eq") upgradeType = utEq; else if (arg == "--always") upgradeType = utAlways; - else throw UsageError(format("unknown flag '%1%'") % arg); + else throw UsageError("unknown flag '%1%'", arg); } upgradeDerivations(globals, opArgs, upgradeType); @@ -637,7 +637,7 @@ static void setMetaFlag(EvalState & state, DrvInfo & drv, static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() < 2) throw UsageError("not enough arguments to '--set-flag'"); @@ -680,7 +680,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { string arg = *i++; if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; - else throw UsageError(format("unknown flag '%1%'") % arg); + else throw UsageError("unknown flag '%1%'", arg); } DrvInfos elems; @@ -695,13 +695,13 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) drv.setName(globals.forceName); if (drv.queryDrvPath() != "") { - std::vector<StorePathWithOutputs> paths{globals.state->store->parseStorePath(drv.queryDrvPath())}; + std::vector<StorePathWithOutputs> paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}}; printMissing(globals.state->store, paths); if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); } else { printMissing(globals.state->store, - {globals.state->store->parseStorePath(drv.queryOutPath())}); + {{globals.state->store->parseStorePath(drv.queryOutPath())}}); if (globals.dryRun) return; globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); } @@ -718,28 +718,39 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, while (true) { string lockToken = optimisticLockProfile(profile); - DrvInfos installedElems = queryInstalled(*globals.state, profile); - DrvInfos newElems; + DrvInfos workingElems = queryInstalled(*globals.state, profile); - for (auto & i : installedElems) { - DrvName drvName(i.queryName()); - bool found = false; - for (auto & j : selectors) - /* !!! the repeated calls to followLinksToStorePath() - are expensive, should pre-compute them. */ - if ((isPath(j) && globals.state->store->parseStorePath(i.queryOutPath()) == globals.state->store->followLinksToStorePath(j)) - || DrvName(j).matches(drvName)) - { - printInfo("uninstalling '%s'", i.queryName()); - found = true; - break; - } - if (!found) newElems.push_back(i); + for (auto & selector : selectors) { + DrvInfos::iterator split = workingElems.begin(); + if (isPath(selector)) { + StorePath selectorStorePath = globals.state->store->followLinksToStorePath(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorStorePath, globals](auto &elem) { + return selectorStorePath != globals.state->store->parseStorePath(elem.queryOutPath()); + } + ); + } else { + DrvName selectorName(selector); + split = std::partition( + workingElems.begin(), workingElems.end(), + [&selectorName](auto &elem){ + DrvName elemName(elem.queryName()); + return !selectorName.matches(elemName); + } + ); + } + if (split == workingElems.end()) + warn("selector '%s' matched no installed derivations", selector); + for (auto removedElem = split; removedElem != workingElems.end(); removedElem++) { + printInfo("uninstalling '%s'", removedElem->queryName()); + } + workingElems.erase(split, workingElems.end()); } if (globals.dryRun) return; - if (createUserEnv(*globals.state, newElems, + if (createUserEnv(*globals.state, workingElems, profile, settings.envKeepDerivations, lockToken)) break; } } @@ -748,7 +759,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); uninstallDerivations(globals, opArgs, globals.profile); } @@ -861,7 +872,11 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems) auto placeholder = metaObj.placeholder(j); Value * v = i.queryMeta(j); if (!v) { - printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); + logError({ + .name = "Invalid meta attribute", + .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) + }); placeholder.write(nullptr); } else { PathSet context; @@ -911,7 +926,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (arg == "--attr" || arg == "-A") attrPath = needArg(i, opFlags, arg); else - throw UsageError(format("unknown flag '%1%'") % arg); + throw UsageError("unknown flag '%1%'", arg); } if (printAttrPath && source != sAvailable) @@ -1112,7 +1127,12 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) attrs2["name"] = j; Value * v = i.queryMeta(j); if (!v) - printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); + logError({ + .name = "Invalid meta attribute", + .hint = hintfmt( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j) + }); else { if (v->type == tString) { attrs2["type"] = "string"; @@ -1177,9 +1197,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + throw UsageError("exactly one argument expected"); Path profile = absPath(opArgs.front()); Path profileLink = getHome() + "/.nix-profile"; @@ -1207,10 +1227,10 @@ static void switchGeneration(Globals & globals, int dstGen) if (!dst) { if (dstGen == prevGen) - throw Error(format("no generation older than the current (%1%) exists") - % curGen); + throw Error("no generation older than the current (%1%) exists", + curGen); else - throw Error(format("generation %1% does not exist") % dstGen); + throw Error("generation %1% does not exist", dstGen); } printInfo(format("switching from generation %1% to %2%") @@ -1225,13 +1245,13 @@ static void switchGeneration(Globals & globals, int dstGen) static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 1) - throw UsageError(format("exactly one argument expected")); + throw UsageError("exactly one argument expected"); int dstGen; if (!string2Int(opArgs.front(), dstGen)) - throw UsageError(format("expected a generation number")); + throw UsageError("expected a generation number"); switchGeneration(globals, dstGen); } @@ -1240,9 +1260,9 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + throw UsageError("no arguments expected"); switchGeneration(globals, prevGen); } @@ -1251,9 +1271,9 @@ static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() != 0) - throw UsageError(format("no arguments expected")); + throw UsageError("no arguments expected"); PathLocks lock; lockProfile(lock, globals.profile); @@ -1278,7 +1298,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs) { if (opFlags.size() > 0) - throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + throw UsageError("unknown flag '%1%'", opFlags.front()); if (opArgs.size() == 1 && opArgs.front() == "old") { deleteOldGenerations(globals.profile, globals.dryRun); @@ -1286,18 +1306,18 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { if(opArgs.front().size() < 2) - throw Error(format("invalid number of generations ‘%1%’") % opArgs.front()); + throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); int max; if (!string2Int(str_max, max) || max == 0) - throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front()); + throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); } else { std::set<unsigned int> gens; for (auto & i : opArgs) { unsigned int n; if (!string2Int(i, n)) - throw UsageError(format("invalid generation number '%1%'") % i); + throw UsageError("invalid generation number '%1%'", i); gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); @@ -1428,26 +1448,15 @@ static int _main(int argc, char * * argv) if (globals.profile == "") globals.profile = getEnv("NIX_PROFILE").value_or(""); - if (globals.profile == "") { - Path profileLink = getHome() + "/.nix-profile"; - try { - if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); - } - globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); - } catch (Error &) { - globals.profile = profileLink; - } - } + if (globals.profile == "") + globals.profile = getDefaultProfile(); op(globals, opFlags, opArgs); globals.state->printStats(); + logger->stop(); + return 0; } } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 717431b7a..8e7f09e12 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -15,6 +15,8 @@ namespace nix { DrvInfos queryInstalled(EvalState & state, const Path & userEnv) { DrvInfos elems; + if (pathExists(userEnv + "/manifest.json")) + throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); Path manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; @@ -132,7 +134,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Realise the resulting store expression. */ debug("building user environment"); std::vector<StorePathWithOutputs> topLevelDrvs; - topLevelDrvs.push_back(StorePathWithOutputs{topLevelDrv.clone()}); + topLevelDrvs.push_back({topLevelDrv}); state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ @@ -144,7 +146,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Path lockTokenCur = optimisticLockProfile(profile); if (lockToken != lockTokenCur) { - printError(format("profile '%1%' changed while we were busy; restarting") % profile); + printInfo("profile '%1%' changed while we were busy; restarting", profile); return false; } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 5a886d69d..bf353677a 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "store-api.hh" #include "common-eval-args.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include <map> #include <iostream> @@ -39,7 +39,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, state.eval(e, vRoot); for (auto & i : attrPaths) { - Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot).first); state.forceValue(v); PathSet context; @@ -66,7 +66,7 @@ void processExpr(EvalState & state, const Strings & attrPaths, /* What output do we want? */ string outputName = i.queryOutputName(); if (outputName == "") - throw Error(format("derivation '%1%' lacks an 'outputName' attribute ") % drvPath); + throw Error("derivation '%1%' lacks an 'outputName' attribute ", drvPath); if (gcRoot == "") printGCWarning(); @@ -166,7 +166,7 @@ static int _main(int argc, char * * argv) if (findFile) { for (auto & i : files) { Path p = state->findFile(i); - if (p == "") throw Error(format("unable to find '%1%'") % i); + if (p == "") throw Error("unable to find '%1%'", i); std::cout << p << std::endl; } return 0; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index cc0891811..55b72bda6 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -1,13 +1,13 @@ #include "hash.hh" #include "shared.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" #include "common-eval-args.hh" #include "attr-path.hh" -#include "legacy.hh" #include "finally.hh" +#include "../nix/legacy.hh" #include "progress-bar.hh" #include "tarfile.hh" @@ -37,11 +37,11 @@ string resolveMirrorUri(EvalState & state, string uri) auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); if (mirrorList == vMirrors.attrs->end()) - throw Error(format("unknown mirror name '%1%'") % mirrorName); + throw Error("unknown mirror name '%1%'", mirrorName); state.forceList(*mirrorList->value); if (mirrorList->value->listSize() < 1) - throw Error(format("mirror URI '%1%' did not expand to anything") % uri); + throw Error("mirror URI '%1%' did not expand to anything", uri); string mirror = state.forceString(*mirrorList->value->listElems()[0]); return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); @@ -73,7 +73,7 @@ static int _main(int argc, char * * argv) string s = getArg(*arg, arg, end); ht = parseHashType(s); if (ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); + throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--print-path") printPath = true; @@ -120,7 +120,7 @@ static int _main(int argc, char * * argv) Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); Value vRoot; state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); state->forceAttrs(v); /* Extract the URI. */ @@ -151,7 +151,7 @@ static int _main(int argc, char * * argv) if (name.empty()) name = baseNameOf(uri); if (name.empty()) - throw Error(format("cannot figure out file name for '%1%'") % uri); + throw Error("cannot figure out file name for '%1%'", uri); /* If an expected hash is given, the file may already exist in the store. */ @@ -159,7 +159,8 @@ static int _main(int argc, char * * argv) std::optional<StorePath> storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); - storePath = store->makeFixedOutputPath(unpack, expectedHash, name); + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(recursive, expectedHash, name); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -180,9 +181,9 @@ static int _main(int argc, char * * argv) FdSink sink(fd.get()); - DownloadRequest req(actualUri); + FileTransferRequest req(actualUri); req.decompress = false; - getDownloader()->download(std::move(req), sink); + getFileTransfer()->download(std::move(req), sink); } /* Optionally unpack the file. */ @@ -206,15 +207,17 @@ static int _main(int argc, char * * argv) hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); if (expectedHash != Hash(ht) && expectedHash != hash) - throw Error(format("hash mismatch for '%1%'") % uri); + throw Error("hash mismatch for '%1%'", uri); + + const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; /* Copy the file to the Nix store. FIXME: if RemoteStore implemented addToStoreFromDump() and downloadFile() supported a sink, we could stream the download directly into the Nix store. */ - storePath = store->addToStore(name, tmpFile, unpack, ht); + storePath = store->addToStore(name, tmpFile, recursive, ht); - assert(*storePath == store->makeFixedOutputPath(unpack, hash, name)); + assert(*storePath == store->makeFixedOutputPath(recursive, hash, name)); } stopProgressBar(); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 667d917f5..8b699f39b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -54,13 +54,13 @@ void printDotGraph(ref<Store> store, StorePathSet && roots) while (!workList.empty()) { auto path = std::move(workList.extract(workList.begin()).value()); - if (!doneSet.insert(path.clone()).second) continue; + if (!doneSet.insert(path).second) continue; cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { - workList.insert(p.clone()); + workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); } } diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 347708851..8ca5c9c8d 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -65,7 +65,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) while (!workList.empty()) { auto path = std::move(workList.extract(workList.begin()).value()); - ret = doneSet.insert(path.clone()); + ret = doneSet.insert(path); if (ret.second == false) continue; auto info = store->queryPathInfo(path); @@ -73,7 +73,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) for (auto & p : info->references) { if (p != path) { - workList.insert(p.clone()); + workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 45e152c47..5c5afd5ec 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,7 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "graphml.hh" -#include "legacy.hh" +#include "../nix/legacy.hh" #include <iostream> #include <algorithm> @@ -49,11 +49,11 @@ ref<LocalStore> ensureLocalStore() static StorePath useDeriver(const StorePath & path) { - if (path.isDerivation()) return path.clone(); + if (path.isDerivation()) return path; auto info = store->queryPathInfo(path); if (!info->deriver) throw Error("deriver of path '%s' is not known", store->printStorePath(path)); - return info->deriver->clone(); + return *info->deriver; } @@ -124,7 +124,7 @@ static void opRealise(Strings opFlags, Strings opArgs) else if (i == "--repair") buildMode = bmRepair; else if (i == "--check") buildMode = bmCheck; else if (i == "--ignore-unknown") ignoreUnknown = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); std::vector<StorePathWithOutputs> paths; for (auto & i : opArgs) @@ -174,11 +174,11 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - bool recursive = false; + auto recursive = FileIngestionMethod::Flat; for (auto & i : opFlags) - if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag '%1%'") % i); + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); @@ -194,14 +194,14 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - bool recursive = false; + auto recursive = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = true; - else throw UsageError(format("unknown flag '%1%'") % i); + if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) - throw UsageError(format("'--print-fixed-path' requires three arguments")); + throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); HashType hashAlgo = parseHashType(*i++); @@ -214,31 +214,25 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise) { - if (forceRealise) realisePath(storePath); + if (forceRealise) realisePath({storePath}); if (useOutput && storePath.isDerivation()) { auto drv = store->derivationFromPath(storePath); StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path.clone()); + outputs.insert(i.second.path); return outputs; } - else return singleton(storePath.clone()); + else return {storePath}; } /* Some code to print a tree representation of a derivation dependency graph. Topological sorting is used to keep the tree relatively flat. */ - -const string treeConn = "+---"; -const string treeLine = "| "; -const string treeNull = " "; - - static void printTree(const StorePath & path, const string & firstPad, const string & tailPad, StorePathSet & done) { - if (!done.insert(path.clone()).second) { + if (!done.insert(path).second) { cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); return; } @@ -254,10 +248,11 @@ static void printTree(const StorePath & path, auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); - for (auto i = sorted.begin(); i != sorted.end(); ++i) { - auto j = i; ++j; - printTree(*i, tailPad + treeConn, - j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + for (const auto &[n, i] : enumerate(sorted)) { + bool last = n + 1 == sorted.size(); + printTree(i, + tailPad + (last ? treeLast : treeConn), + tailPad + (last ? treeNull : treeLine), done); } } @@ -301,9 +296,9 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--use-output" || i == "-u") useOutput = true; else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; else if (i == "--include-outputs") includeOutputs = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (prev != qDefault && prev != query) - throw UsageError(format("query type '%1%' conflicts with earlier flag") % i); + throw UsageError("query type '%1%' conflicts with earlier flag", i); } if (query == qDefault) query = qOutputs; @@ -315,7 +310,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qOutputs: { for (auto & i : opArgs) { auto i2 = store->followLinksToStorePath(i); - if (forceRealise) realisePath(i2); + if (forceRealise) realisePath({i2}); Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) cout << fmt("%1%\n", store->printStorePath(j.second.path)); @@ -334,13 +329,13 @@ static void opQuery(Strings opFlags, Strings opArgs) if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { for (auto & p : store->queryPathInfo(j)->references) - paths.insert(p.clone()); + paths.insert(p); } else if (query == qReferrers) { StorePathSet tmp; store->queryReferrers(j, tmp); for (auto & i : tmp) - paths.insert(i.clone()); + paths.insert(i); } else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); } @@ -378,7 +373,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32)); + cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -396,7 +391,7 @@ static void opQuery(Strings opFlags, Strings opArgs) StorePathSet roots; for (auto & i : opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) - roots.insert(j.clone()); + roots.insert(j); printDotGraph(ref<Store>(store), std::move(roots)); break; } @@ -405,7 +400,7 @@ static void opQuery(Strings opFlags, Strings opArgs) StorePathSet roots; for (auto & i : opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) - roots.insert(j.clone()); + roots.insert(j); printGraphML(ref<Store>(store), std::move(roots)); break; } @@ -420,7 +415,7 @@ static void opQuery(Strings opFlags, Strings opArgs) StorePathSet args; for (auto & i : opArgs) for (auto & p : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) - args.insert(p.clone()); + args.insert(p); StorePathSet referrers; store->computeFSClosure( @@ -449,7 +444,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath)); /* Print each environment variable in the derivation in a format - that can be sourced by the shell. */ + * that can be sourced by the shell. */ for (auto & i : drv.env) cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); @@ -487,10 +482,10 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opArgs.empty()) { for (auto & i : opArgs) - cout << store->makeValidityRegistration(singleton(store->followLinksToStorePath(i)), true, true); + cout << store->makeValidityRegistration({store->followLinksToStorePath(i)}, true, true); } else { for (auto & i : store->queryAllValidPaths()) - cout << store->makeValidityRegistration(singleton(i), true, true); + cout << store->makeValidityRegistration({i}, true, true); } } @@ -536,7 +531,7 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--reregister") reregister = true; else if (i == "--hash-given") hashGiven = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -550,7 +545,7 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--print-invalid") printInvalid = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) { auto path = store->followLinksToStorePath(i); @@ -577,12 +572,11 @@ static void opGC(Strings opFlags, Strings opArgs) if (*i == "--print-roots") printRoots = true; else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; - else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; else if (*i == "--max-freed") { long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true); options.maxFreed = maxFreed >= 0 ? maxFreed : 0; } - else throw UsageError(format("bad sub-operation '%1%' in GC") % *i); + else throw UsageError("bad sub-operation '%1%' in GC", *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -592,7 +586,7 @@ static void opGC(Strings opFlags, Strings opArgs) // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) - roots2.emplace(link, target.clone()); + roots2.emplace(link, target); for (auto & [link, target] : roots2) std::cout << link << " -> " << store->printStorePath(target) << "\n"; } @@ -618,7 +612,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--ignore-liveness") options.ignoreLiveness = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); @@ -656,7 +650,7 @@ static void opRestore(Strings opFlags, Strings opArgs) static void opExport(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + throw UsageError("unknown flag '%1%'", i); StorePathSet paths; @@ -672,7 +666,7 @@ static void opExport(Strings opFlags, Strings opArgs) static void opImport(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -707,10 +701,13 @@ static void opVerify(Strings opFlags, Strings opArgs) for (auto & i : opFlags) if (i == "--check-contents") checkContents = true; else if (i == "--repair") repair = Repair; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (store->verifyStore(checkContents, repair)) { - printError("warning: not all errors were fixed"); + logWarning({ + .name = "Store consistency", + .description = "not all errors were fixed" + }); throw Exit(1); } } @@ -732,9 +729,14 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { - printError( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), info->narHash.to_string(), current.first.to_string()); + logError({ + .name = "Hash mismatch", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), + info->narHash.to_string(Base32, true), + current.first.to_string(Base32, true)) + }); status = 1; } } @@ -770,7 +772,7 @@ static void opServe(Strings opFlags, Strings opArgs) bool writeAllowed = false; for (auto & i : opFlags) if (i == "--write") writeAllowed = true; - else throw UsageError(format("unknown flag '%1%'") % i); + else throw UsageError("unknown flag '%1%'", i); if (!opArgs.empty()) throw UsageError("no arguments expected"); @@ -828,7 +830,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector<StorePathWithOutputs> paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.emplace_back(path.clone()); + paths2.push_back({path}); unsigned long long downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths2, @@ -838,10 +840,10 @@ static void opServe(Strings opFlags, Strings opArgs) if (!willSubstitute.empty()) try { std::vector<StorePathWithOutputs> subs; - for (auto & p : willSubstitute) subs.emplace_back(p.clone()); + for (auto & p : willSubstitute) subs.push_back({p}); store->buildPaths(subs); } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); + logWarning(e.info()); } } @@ -862,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } @@ -893,7 +895,7 @@ static void opServe(Strings opFlags, Strings opArgs) std::vector<StorePathWithOutputs> paths; for (auto & s : readStrings<Strings>(in)) - paths.emplace_back(store->parsePathWithOutputs(s)); + paths.push_back(store->parsePathWithOutputs(s)); getBuildSettings(); @@ -968,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) } default: - throw Error(format("unknown serve command %1%") % cmd); + throw Error("unknown serve command %1%", cmd); } out.flush(); @@ -979,7 +981,7 @@ static void opServe(Strings opFlags, Strings opArgs) static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) { for (auto & i : opFlags) - throw UsageError(format("unknown flag '%1%'") % i); + throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) throw UsageError("three arguments expected"); auto i = opArgs.begin(); @@ -1104,6 +1106,8 @@ static int _main(int argc, char * * argv) op(opFlags, opArgs); + logger->stop(); + return 0; } } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 139db3657..f43f774c1 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -14,12 +14,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand { expectArg("path", &path); - mkFlag() - .longName("name") - .shortName('n') - .description("name component of the store path") - .labels({"name"}) - .dest(&namePart); + addFlag({ + .longName = "name", + .shortName = 'n', + .description = "name component of the store path", + .labels = {"name"}, + .handler = {&namePart}, + }); } std::string description() override @@ -33,6 +34,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand }; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { if (!namePart) namePart = baseNameOf(path); @@ -42,15 +45,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(true, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); - if (!dryRun) - store->addToStore(info, sink.s); + if (!dryRun) { + auto source = StringSource { *sink.s }; + store->addToStore(info, source); + } - std::cout << fmt("%s\n", store->printStorePath(info.path)); + logger->stdout("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/build.cc b/src/nix/build.cc index 3c9d2df39..850e09ce8 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -5,23 +5,25 @@ using namespace nix; -struct CmdBuild : MixDryRun, InstallablesCommand +struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile { Path outLink = "result"; CmdBuild() { - mkFlag() - .longName("out-link") - .shortName('o') - .description("path of the symlink to the build result") - .labels({"path"}) - .dest(&outLink); + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "path of the symlink to the build result", + .labels = {"path"}, + .handler = {&outLink}, + }); - mkFlag() - .longName("no-link") - .description("do not create a symlink to the build result") - .set(&outLink, Path("")); + addFlag({ + .longName = "no-link", + .description = "do not create a symlink to the build result", + .handler = {&outLink, Path("")}, + }); } std::string description() override @@ -40,6 +42,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand "To build the build.x86_64-linux attribute from release.nix:", "nix build -f release.nix build.x86_64-linux" }, + Example{ + "To make a profile point at GNU Hello:", + "nix build --profile /tmp/profile nixpkgs.hello" + }, }; } @@ -49,18 +55,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand if (dryRun) return; - for (size_t i = 0; i < buildables.size(); ++i) { - auto & b(buildables[i]); - - if (outLink != "") - for (auto & output : b.outputs) + if (outLink != "") { + for (size_t i = 0; i < buildables.size(); ++i) { + for (auto & output : buildables[i].outputs) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { std::string symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); store2->addPermRoot(output.second, absPath(symlink), true); } + } } + + updateProfile(buildables); } }; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 851f90abd..c82819af8 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -13,9 +13,9 @@ struct MixCat : virtual Args { auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path '%1%' does not exist") % path); + throw Error("path '%1%' does not exist", path); if (st.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' is not a regular file") % path); + throw Error("path '%1%' is not a regular file", path); std::cout << accessor->readFile(path); } @@ -30,9 +30,11 @@ struct CmdCatStore : StoreCommand, MixCat std::string description() override { - return "print the contents of a store file on stdout"; + return "print the contents of a file in the Nix store on stdout"; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { cat(store->getFSAccessor()); @@ -51,9 +53,11 @@ struct CmdCatNar : StoreCommand, MixCat std::string description() override { - return "print the contents of a file inside a NAR file"; + return "print the contents of a file inside a NAR file on stdout"; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); diff --git a/src/nix/command.cc b/src/nix/command.cc index 442bc6c53..d62626c26 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -2,6 +2,9 @@ #include "store-api.hh" #include "derivations.hh" #include "nixexpr.hh" +#include "profiles.hh" + +extern char * * environ; namespace nix { @@ -32,16 +35,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { if (recursive) - mkFlag() - .longName("no-recursive") - .description("apply operation to specified paths only") - .set(&this->recursive, false); + addFlag({ + .longName = "no-recursive", + .description = "apply operation to specified paths only", + .handler = {&this->recursive, false}, + }); else - mkFlag() - .longName("recursive") - .shortName('r') - .description("apply operation to closure of the specified paths") - .set(&this->recursive, true); + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "apply operation to closure of the specified paths", + .handler = {&this->recursive, true}, + }); mkFlag(0, "all", "apply operation to the entire store", &all); } @@ -54,19 +59,19 @@ void StorePathsCommand::run(ref<Store> store) if (installables.size()) throw UsageError("'--all' does not expect arguments"); for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p.clone()); + storePaths.push_back(p); } else { for (auto & p : toStorePaths(store, realiseMode, installables)) - storePaths.push_back(p.clone()); + storePaths.push_back(p); if (recursive) { StorePathSet closure; - store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); + store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false); storePaths.clear(); for (auto & p : closure) - storePaths.push_back(p.clone()); + storePaths.push_back(p); } } @@ -96,4 +101,98 @@ Strings editorFor(const Pos & pos) return args; } +MixProfile::MixProfile() +{ + addFlag({ + .longName = "profile", + .description = "profile to update", + .labels = {"path"}, + .handler = {&profile}, + }); +} + +void MixProfile::updateProfile(const StorePath & storePath) +{ + if (!profile) return; + auto store = getStore().dynamic_pointer_cast<LocalFSStore>(); + if (!store) throw Error("'--profile' is not supported for this Nix store"); + auto profile2 = absPath(*profile); + switchLink(profile2, + createGeneration( + ref<LocalFSStore>(store), + profile2, store->printStorePath(storePath))); +} + +void MixProfile::updateProfile(const Buildables & buildables) +{ + if (!profile) return; + + std::optional<StorePath> result; + + for (auto & buildable : buildables) { + for (auto & output : buildable.outputs) { + if (result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); + result = output.second; + } + } + + if (!result) + throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); + + updateProfile(*result); +} + +MixDefaultProfile::MixDefaultProfile() +{ + profile = getDefaultProfile(); +} + +MixEnvironment::MixEnvironment() : ignoreEnvironment(false) +{ + addFlag({ + .longName = "ignore-environment", + .shortName = 'i', + .description = "clear the entire environment (except those specified with --keep)", + .handler = {&ignoreEnvironment, true}, + }); + + addFlag({ + .longName = "keep", + .shortName = 'k', + .description = "keep specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { keep.insert(s); }}, + }); + + addFlag({ + .longName = "unset", + .shortName = 'u', + .description = "unset specified environment variable", + .labels = {"name"}, + .handler = {[&](std::string s) { unset.insert(s); }}, + }); +} + +void MixEnvironment::setEnviron() { + if (ignoreEnvironment) { + if (!unset.empty()) + throw UsageError("--unset does not make sense with --ignore-environment"); + + for (const auto & var : keep) { + auto val = getenv(var.c_str()); + if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val)); + } + + vectorEnv = stringsToCharPtrs(stringsEnv); + environ = vectorEnv.data(); + } else { + if (!keep.empty()) + throw UsageError("--keep does not make sense without --ignore-environment"); + + for (const auto & var : unset) + unsetenv(var.c_str()); + } +} + } diff --git a/src/nix/command.hh b/src/nix/command.hh index 00eb46903..959d5f19d 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -1,18 +1,18 @@ #pragma once +#include "installables.hh" #include "args.hh" #include "common-eval-args.hh" #include "path.hh" +#include "eval.hh" namespace nix { extern std::string programPath; -struct Value; -class Bindings; -class EvalState; -struct Pos; -class Store; +static constexpr Command::Category catSecondary = 100; +static constexpr Command::Category catUtility = 101; +static constexpr Command::Category catNixInstallation = 102; /* A command that requires a Nix store. */ struct StoreCommand : virtual Command @@ -27,34 +27,7 @@ private: std::shared_ptr<Store> _store; }; -struct Buildable -{ - std::optional<StorePath> drvPath; - std::map<std::string, StorePath> outputs; -}; - -typedef std::vector<Buildable> Buildables; - -struct Installable -{ - virtual ~Installable() { } - - virtual std::string what() = 0; - - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } - - Buildable toBuildable(); - - virtual Value * toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } -}; - -struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs +struct SourceExprCommand : virtual StoreCommand, MixEvalArgs { Path file; @@ -72,7 +45,7 @@ private: std::shared_ptr<EvalState> evalState; - Value * vSourceExpr = 0; + RootValue vSourceExpr; }; enum RealiseMode { Build, NoBuild, DryRun }; @@ -189,4 +162,36 @@ std::set<StorePath> toDerivations(ref<Store> store, filename:lineno. */ Strings editorFor(const Pos & pos); +struct MixProfile : virtual StoreCommand +{ + std::optional<Path> profile; + + MixProfile(); + + /* If 'profile' is set, make it point at 'storePath'. */ + void updateProfile(const StorePath & storePath); + + /* If 'profile' is set, make it point at the store path produced + by 'buildables'. */ + void updateProfile(const Buildables & buildables); +}; + +struct MixDefaultProfile : MixProfile +{ + MixDefaultProfile(); +}; + +struct MixEnvironment : virtual Args { + + StringSet keep, unset; + Strings stringsEnv; + std::vector<char*> vectorEnv; + bool ignoreEnvironment; + + MixEnvironment(); + + /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + void setEnviron(); +}; + } diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 85c777d38..64099f476 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -19,27 +19,32 @@ struct CmdCopy : StorePathsCommand CmdCopy() : StorePathsCommand(true) { - mkFlag() - .longName("from") - .labels({"store-uri"}) - .description("URI of the source Nix store") - .dest(&srcUri); - mkFlag() - .longName("to") - .labels({"store-uri"}) - .description("URI of the destination Nix store") - .dest(&dstUri); - - mkFlag() - .longName("no-check-sigs") - .description("do not require that paths are signed by trusted keys") - .set(&checkSigs, NoCheckSigs); - - mkFlag() - .longName("substitute-on-destination") - .shortName('s') - .description("whether to try substitutes on the destination store (only supported by SSH)") - .set(&substitute, Substitute); + addFlag({ + .longName = "from", + .description = "URI of the source Nix store", + .labels = {"store-uri"}, + .handler = {&srcUri}, + }); + + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri}, + }); + + addFlag({ + .longName = "no-check-sigs", + .description = "do not require that paths are signed by trusted keys", + .handler = {&checkSigs, NoCheckSigs}, + }); + + addFlag({ + .longName = "substitute-on-destination", + .shortName = 's', + .description = "whether to try substitutes on the destination store (only supported by SSH)", + .handler = {&substitute, Substitute}, + }); } std::string description() override @@ -75,6 +80,8 @@ struct CmdCopy : StorePathsCommand }; } + Category category() override { return catSecondary; } + ref<Store> createStore() override { return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); @@ -87,7 +94,7 @@ struct CmdCopy : StorePathsCommand ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, storePathsToSet(storePaths), + copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc new file mode 100644 index 000000000..87039e8a0 --- /dev/null +++ b/src/nix/develop.cc @@ -0,0 +1,344 @@ +#include "eval.hh" +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "progress-bar.hh" + +#include <regex> + +using namespace nix; + +struct Var +{ + bool exported = true; + bool associative = false; + std::string value; // quoted string or array +}; + +struct BuildEnvironment +{ + std::map<std::string, Var> env; + std::string bashFunctions; +}; + +BuildEnvironment readEnvironment(const Path & path) +{ + BuildEnvironment res; + + std::set<std::string> exported; + + debug("reading environment file '%s'", path); + + auto file = readFile(path); + + auto pos = file.cbegin(); + + static std::string varNameRegex = + R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + + static std::regex declareRegex( + "^declare -x (" + varNameRegex + ")" + + R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); + + static std::string simpleStringRegex = + R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + + static std::string quotedStringRegex = + R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + + static std::string indexedArrayRegex = + R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re"; + + static std::regex varRegex( + "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + + /* Note: we distinguish between an indexed and associative array + using the space before the closing parenthesis. Will + undoubtedly regret this some day. */ + static std::regex assocArrayRegex( + "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); + + static std::regex functionRegex( + "^" + varNameRegex + " \\(\\) *\n"); + + while (pos != file.end()) { + + std::smatch match; + + if (std::regex_search(pos, file.cend(), match, declareRegex)) { + pos = match[0].second; + exported.insert(match[1]); + } + + else if (std::regex_search(pos, file.cend(), match, varRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { + pos = match[0].second; + res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); + } + + else if (std::regex_search(pos, file.cend(), match, functionRegex)) { + res.bashFunctions = std::string(pos, file.cend()); + break; + } + + else throw Error("shell environment '%s' has unexpected line '%s'", + path, file.substr(pos - file.cbegin(), 60)); + } + + return res; +} + +const static std::string getEnvSh = + #include "get-env.sh.gen.hh" + ; + +/* Given an existing derivation, return the shell environment as + initialised by stdenv's setup script. We do this by building a + modified derivation with the same dependencies and nearly the same + initial environment variables, that just writes the resulting + environment to a file and exits. */ +StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) +{ + auto drv = store->derivationFromPath(drvPath); + + auto builder = baseNameOf(drv.builder); + if (builder != "bash") + throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); + + auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + + drv.args = {store->printStorePath(getEnvShPath)}; + + /* Remove derivation checks. */ + drv.env.erase("allowedReferences"); + drv.env.erase("allowedRequisites"); + drv.env.erase("disallowedReferences"); + drv.env.erase("disallowedRequisites"); + + /* Rehash and write the derivation. FIXME: would be nice to use + 'buildDerivation', but that's privileged. */ + auto drvName = std::string(drvPath.name()); + assert(hasSuffix(drvName, ".drv")); + drvName.resize(drvName.size() - 4); + drvName += "-env"; + for (auto & output : drv.outputs) + drv.env.erase(output.first); + drv.env["out"] = ""; + drv.env["outputs"] = "out"; + drv.inputSrcs.insert(std::move(getEnvShPath)); + Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + auto shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs.insert_or_assign("out", DerivationOutput { shellOutPath, "", "" }); + drv.env["out"] = store->printStorePath(shellOutPath); + auto shellDrvPath2 = writeDerivation(store, drv, drvName); + + /* Build the derivation. */ + store->buildPaths({{shellDrvPath2}}); + + assert(store->isValidPath(shellOutPath)); + + return shellOutPath; +} + +struct Common : InstallableCommand, MixProfile +{ + std::set<string> ignoreVars{ + "BASHOPTS", + "EUID", + "HOME", // FIXME: don't ignore in pure mode? + "NIX_BUILD_TOP", + "NIX_ENFORCE_PURITY", + "NIX_LOG_FD", + "PPID", + "PWD", + "SHELLOPTS", + "SHLVL", + "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt + "TEMP", + "TEMPDIR", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "UID", + }; + + void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + { + out << "unset shellHook\n"; + + out << "nix_saved_PATH=\"$PATH\"\n"; + + for (auto & i : buildEnvironment.env) { + if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { + if (i.second.associative) + out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); + else { + out << fmt("%s=%s\n", i.first, i.second.value); + if (i.second.exported) + out << fmt("export %s\n", i.first); + } + } + } + + out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; + + out << buildEnvironment.bashFunctions << "\n"; + + // FIXME: set outputs + + out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) + out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); + + out << "eval \"$shellHook\"\n"; + } + + StorePath getShellOutPath(ref<Store> store) + { + auto path = installable->getStorePath(); + if (path && hasSuffix(path->to_string(), "-env")) + return *path; + else { + auto drvs = toDerivations(store, {installable}); + + if (drvs.size() != 1) + throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations", + installable->what(), drvs.size()); + + auto & drvPath = *drvs.begin(); + + return getDerivationEnvironment(store, drvPath); + } + } + + std::pair<BuildEnvironment, std::string> getBuildEnvironment(ref<Store> store) + { + auto shellOutPath = getShellOutPath(store); + + auto strPath = store->printStorePath(shellOutPath); + + updateProfile(shellOutPath); + + return {readEnvironment(strPath), strPath}; + } +}; + +struct CmdDevelop : Common, MixEnvironment +{ + std::vector<std::string> command; + + CmdDevelop() + { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed insted of an interactive shell", + .labels = {"command", "args"}, + .handler = {[&](std::vector<std::string> ss) { + if (ss.empty()) throw UsageError("--command requires at least one argument"); + command = ss; + }} + }); + } + + std::string description() override + { + return "run a bash shell that provides the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To get the build environment of GNU hello:", + "nix develop nixpkgs.hello" + }, + Example{ + "To store the build environment in a profile:", + "nix develop --profile /tmp/my-shell nixpkgs.hello" + }, + Example{ + "To use a build environment previously recorded in a profile:", + "nix develop /tmp/my-shell" + }, + }; + } + + void run(ref<Store> store) override + { + auto [buildEnvironment, gcroot] = getBuildEnvironment(store); + + auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); + + std::ostringstream ss; + makeRcScript(buildEnvironment, ss); + + ss << fmt("rm -f '%s'\n", rcFilePath); + + if (!command.empty()) { + std::vector<std::string> args; + for (auto s : command) + args.push_back(shellEscape(s)); + ss << fmt("exec %s\n", concatStringsSep(" ", args)); + } + + writeFull(rcFileFd.get(), ss.str()); + + stopProgressBar(); + + auto shell = getEnv("SHELL").value_or("bash"); + + setEnviron(); + // prevent garbage collection until shell exits + setenv("NIX_GCROOT", gcroot.data(), 1); + + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + + restoreAffinity(); + restoreSignals(); + + execvp(shell.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing shell '%s'", shell); + } +}; + +struct CmdPrintDevEnv : Common +{ + std::string description() override + { + return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; + } + + Examples examples() override + { + return { + Example{ + "To apply the build environment of GNU hello to the current shell:", + ". <(nix print-dev-env nixpkgs.hello)" + }, + }; + } + + Category category() override { return catUtility; } + + void run(ref<Store> store) override + { + auto buildEnvironment = getBuildEnvironment(store).first; + + stopProgressBar(); + + makeRcScript(buildEnvironment, std::cout); + } +}; + +static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env"); +static auto r2 = registerCommand<CmdDevelop>("develop"); diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 0aa634d6e..82e92cdd0 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -40,9 +40,11 @@ struct CmdDoctor : StoreCommand std::string description() override { - return "check your system for potential problems and print a PASS or FAIL for each check."; + return "check your system for potential problems and print a PASS or FAIL for each check"; } + Category category() override { return catNixInstallation; } + void run(ref<Store> store) override { logger->log("Running checks against store uri: " + store->getUri()); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index bb741b572..e1de71bf8 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -20,6 +20,8 @@ struct CmdDumpPath : StorePathCommand }; } + Category category() override { return catUtility; } + void run(ref<Store> store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index ca410cd1f..067d3a973 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -25,13 +25,21 @@ struct CmdEdit : InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref<Store> store) override { auto state = getEvalState(); - auto v = installable->toValue(*state); + auto [v, pos] = installable->toValue(*state); + + try { + pos = findDerivationFilename(*state, *v, installable->what()); + } catch (NoPositionInfo &) { + } - Pos pos = findDerivationFilename(*state, *v, installable->what()); + if (pos == noPos) + throw Error("cannot find position information for '%s", installable->what()); stopProgressBar(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 276fdf003..26e98ac2a 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -45,6 +45,8 @@ struct CmdEval : MixJSON, InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref<Store> store) override { if (raw && json) @@ -52,19 +54,18 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto v = installable->toValue(*state); + auto v = installable->toValue(*state).first; PathSet context; - stopProgressBar(); - if (raw) { + stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); } else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - std::cout << *v << "\n"; + logger->stdout("%s", *v); } } }; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh new file mode 100644 index 000000000..a25ec43a9 --- /dev/null +++ b/src/nix/get-env.sh @@ -0,0 +1,9 @@ +set -e +if [ -e .attrs.sh ]; then source .attrs.sh; fi +export IN_NIX_SHELL=impure +export dontAddDisableDepTrack=1 +if [[ -n $stdenv ]]; then + source $stdenv/setup +fi +export > $out +set >> $out diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0cc523f50..0dd4998d9 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,23 +9,20 @@ using namespace nix; struct CmdHash : Command { - enum Mode { mFile, mPath }; - Mode mode; + FileIngestionMethod mode; Base base = SRI; bool truncate = false; HashType ht = htSHA256; std::vector<std::string> paths; std::optional<std::string> modulus; - CmdHash(Mode mode) : mode(mode) + CmdHash(FileIngestionMethod mode) : mode(mode) { mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); mkFlag(0, "base16", "print hash in base-16", &base, Base16); - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") @@ -38,11 +35,18 @@ struct CmdHash : Command std::string description() override { - return mode == mFile - ? "print cryptographic hash of a regular file" - : "print cryptographic hash of the NAR serialisation of a path"; + const char* d; + switch (mode) { + case FileIngestionMethod::Flat: + d = "print cryptographic hash of a regular file"; + case FileIngestionMethod::Recursive: + d = "print cryptographic hash of the NAR serialisation of a path"; + }; + return d; } + Category category() override { return catUtility; } + void run() override { for (auto path : paths) { @@ -53,21 +57,24 @@ struct CmdHash : Command else hashSink = std::make_unique<HashSink>(ht); - if (mode == mFile) + switch (mode) { + case FileIngestionMethod::Flat: readFile(path, *hashSink); - else + break; + case FileIngestionMethod::Recursive: dumpPath(path, *hashSink); + break; + } Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - h.to_string(base, base == SRI); + logger->stdout(h.to_string(base, base == SRI)); } } }; -static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(CmdHash::mFile); }); -static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(CmdHash::mPath); }); +static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); }); +static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); }); struct CmdToBase : Command { @@ -77,9 +84,7 @@ struct CmdToBase : Command CmdToBase(Base base) : base(base) { - mkFlag() - .longName("type") - .mkHashTypeFlag(&ht); + addFlag(Flag::mkHashTypeFlag("type", &ht)); expectArgs("strings", &args); } @@ -92,10 +97,12 @@ struct CmdToBase : Command "SRI"); } + Category category() override { return catUtility; } + void run() override { for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); + logger->stdout(Hash(s, ht).to_string(base, base == SRI)); } }; @@ -126,7 +133,7 @@ static int compatNixHash(int argc, char * * argv) string s = getArg(*arg, arg, end); ht = parseHashType(s); if (ht == htUnknown) - throw UsageError(format("unknown hash type '%1%'") % s); + throw UsageError("unknown hash type '%1%'", s); } else if (*arg == "--to-base16") op = opTo16; else if (*arg == "--to-base32") op = opTo32; @@ -138,7 +145,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 8ce6bd06e..708a0dc88 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -12,26 +12,28 @@ namespace nix { + SourceExprCommand::SourceExprCommand() { - mkFlag() - .shortName('f') - .longName("file") - .label("file") - .description("evaluate FILE rather than the default") - .dest(&file); + addFlag({ + .longName = "file", + .shortName = 'f', + .description = "evaluate FILE rather than the default", + .labels = {"file"}, + .handler = {&file} + }); } Value * SourceExprCommand::getSourceExpr(EvalState & state) { - if (vSourceExpr) return vSourceExpr; + if (vSourceExpr) return *vSourceExpr; auto sToplevel = state.symbols.create("_toplevel"); - vSourceExpr = state.allocValue(); + vSourceExpr = allocRootValue(state.allocValue()); if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); + state.evalFile(lookupFileArg(state, file), **vSourceExpr); else { @@ -39,9 +41,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vSourceExpr, 1024); + state.mkAttrs(**vSourceExpr, 1024); - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); + mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); std::unordered_set<std::string> seen; @@ -52,7 +54,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); Value * v2 = state.allocValue(); mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), + mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), state.getBuiltin("import"), *v2); }; @@ -66,10 +68,10 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) } else addEntry(i.first); - vSourceExpr->attrs->sort(); + (*vSourceExpr)->attrs->sort(); } - return vSourceExpr; + return *vSourceExpr; } ref<EvalState> SourceExprCommand::getEvalState() @@ -100,15 +102,20 @@ struct InstallableStorePath : Installable Buildables toBuildables() override { std::map<std::string, StorePath> outputs; - outputs.insert_or_assign("out", storePath.clone()); + outputs.insert_or_assign("out", storePath); Buildable b{ - .drvPath = storePath.isDerivation() ? storePath.clone() : std::optional<StorePath>(), + .drvPath = storePath.isDerivation() ? storePath : std::optional<StorePath>(), .outputs = std::move(outputs) }; Buildables bs; bs.push_back(std::move(b)); return bs; } + + std::optional<StorePath> getStorePath() override + { + return storePath; + } }; struct InstallableValue : Installable @@ -121,7 +128,7 @@ struct InstallableValue : Installable { auto state = cmd.getEvalState(); - auto v = toValue(*state); + auto v = toValue(*state).first; Bindings & autoArgs = *cmd.getAutoArgs(*state); @@ -134,7 +141,7 @@ struct InstallableValue : Installable for (auto & drv : drvs) { Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; - drvPaths.insert(b.drvPath->clone()); + drvPaths.insert(*b.drvPath); auto outputName = drv.queryOutputName(); if (outputName == "") @@ -148,10 +155,10 @@ struct InstallableValue : Installable // Hack to recognize .all: if all drvs have the same drvPath, // merge the buildables. if (drvPaths.size() == 1) { - Buildable b{.drvPath = drvPaths.begin()->clone()}; + Buildable b{.drvPath = *drvPaths.begin()}; for (auto & b2 : res) for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second.clone()); + b.outputs.insert_or_assign(output.first, output.second); Buildables bs; bs.push_back(std::move(b)); return bs; @@ -169,11 +176,11 @@ struct InstallableExpr : InstallableValue std::string what() override { return text; } - Value * toValue(EvalState & state) override + std::pair<Value *, Pos> toValue(EvalState & state) override { auto v = state.allocValue(); state.eval(state.parseExprFromString(text, absPath(".")), *v); - return v; + return {v, noPos}; } }; @@ -187,16 +194,16 @@ struct InstallableAttrPath : InstallableValue std::string what() override { return attrPath; } - Value * toValue(EvalState & state) override + std::pair<Value *, Pos> toValue(EvalState & state) override { auto source = cmd.getSourceExpr(state); Bindings & autoArgs = *cmd.getAutoArgs(state); - Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); + auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; state.forceValue(*v); - return v; + return {v, noPos}; } }; @@ -266,7 +273,7 @@ Buildables build(ref<Store> store, RealiseMode mode, pathsToBuild.push_back({*b.drvPath, outputNames}); } else for (auto & output : b.outputs) - pathsToBuild.push_back({output.second.clone()}); + pathsToBuild.push_back({output.second}); buildables.push_back(std::move(b)); } } @@ -286,7 +293,7 @@ StorePathSet toStorePaths(ref<Store> store, RealiseMode mode, for (auto & b : build(store, mode, installables)) for (auto & output : b.outputs) - outPaths.insert(output.second.clone()); + outPaths.insert(output.second); return outPaths; } @@ -299,7 +306,7 @@ StorePath toStorePath(ref<Store> store, RealiseMode mode, if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); - return paths.begin()->clone(); + return *paths.begin(); } StorePathSet toDerivations(ref<Store> store, @@ -317,10 +324,10 @@ StorePathSet toDerivations(ref<Store> store, if (derivers.empty()) throw Error("'%s' does not have a known deriver", i->what()); // FIXME: use all derivers? - drvPaths.insert(derivers.begin()->clone()); + drvPaths.insert(*derivers.begin()); } } else - drvPaths.insert(b.drvPath->clone()); + drvPaths.insert(*b.drvPath); } return drvPaths; diff --git a/src/nix/installables.hh b/src/nix/installables.hh new file mode 100644 index 000000000..503984220 --- /dev/null +++ b/src/nix/installables.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "util.hh" +#include "path.hh" +#include "eval.hh" + +#include <optional> + +namespace nix { + +struct Buildable +{ + std::optional<StorePath> drvPath; + std::map<std::string, StorePath> outputs; +}; + +typedef std::vector<Buildable> Buildables; + +struct Installable +{ + virtual ~Installable() { } + + virtual std::string what() = 0; + + virtual Buildables toBuildables() + { + throw Error("argument '%s' cannot be built", what()); + } + + Buildable toBuildable(); + + virtual std::pair<Value *, Pos> toValue(EvalState & state) + { + throw Error("argument '%s' cannot be evaluated", what()); + } + + /* Return a value only if this installable is a store path or a + symlink to it. */ + virtual std::optional<StorePath> getStorePath() + { + return {}; + } +}; + +} diff --git a/src/nix/local.mk b/src/nix/local.mk index 51dad101f..b057b7cc6 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,7 +15,9 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libnixrust +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain + +nix_LIBS = libexpr libmain libfetchers libstore libutil nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system @@ -25,3 +27,5 @@ $(foreach name, \ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh + +src/nix/develop.cc: src/nix/get-env.sh.gen.hh diff --git a/src/nix/log.cc b/src/nix/log.cc index 795991cb7..3fe22f6c2 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -31,6 +31,8 @@ struct CmdLog : InstallableCommand }; } + Category category() override { return catSecondary; } + void run(ref<Store> store) override { settings.readOnlyMode = true; diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3ef1f2750..d2157f2d4 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -34,16 +34,14 @@ struct MixLs : virtual Args, MixJSON (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - std::cout << - (format("%s %20d %s") % tp % st.fileSize % relPath); + auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) - std::cout << " -> " << accessor->readLink(curPath) - ; - std::cout << "\n"; + line += " -> " + accessor->readLink(curPath); + logger->stdout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - std::cout << relPath << "\n"; + logger->stdout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) @@ -65,7 +63,7 @@ struct MixLs : virtual Args, MixJSON auto st = accessor->stat(path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("path '%1%' does not exist") % path); + throw Error("path '%1%' does not exist", path); doPath(st, path, st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); @@ -102,9 +100,11 @@ struct CmdLsStore : StoreCommand, MixLs std::string description() override { - return "show information about a store path"; + return "show information about a path in the Nix store"; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { list(store->getFSAccessor()); @@ -133,12 +133,14 @@ struct CmdLsNar : Command, MixLs std::string description() override { - return "show information about the contents of a NAR file"; + return "show information about a path inside a NAR file"; } + Category category() override { return catUtility; } + void run() override { - list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true)))); + list(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 41d44f431..203901168 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,8 +8,9 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "download.hh" +#include "filetransfer.hh" #include "finally.hh" +#include "loggers.hh" #include <sys/types.h> #include <sys/socket.h> @@ -55,18 +56,26 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; bool useNet = true; + bool refresh = false; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { - mkFlag() - .longName("help") - .description("show usage information") - .handler([&]() { showHelpAndExit(); }); - - mkFlag() - .longName("help-config") - .description("show configuration options") - .handler([&]() { + categories.clear(); + categories[Command::catDefault] = "Main commands"; + categories[catSecondary] = "Infrequently used commands"; + categories[catUtility] = "Utility/scripting commands"; + categories[catNixInstallation] = "Commands for upgrading or troubleshooting your Nix installation"; + + addFlag({ + .longName = "help", + .description = "show usage information", + .handler = {[&]() { showHelpAndExit(); }}, + }); + + addFlag({ + .longName = "help-config", + .description = "show configuration options", + .handler = {[&]() { std::cout << "The following configuration options are available:\n\n"; Table2 tbl; std::map<std::string, Config::SettingInfo> settings; @@ -75,23 +84,35 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs tbl.emplace_back(s.first, s.second.description); printTable(std::cout, tbl); throw Exit(); - }); - - mkFlag() - .longName("print-build-logs") - .shortName('L') - .description("print full build logs on stderr") - .set(&printBuildLogs, true); - - mkFlag() - .longName("version") - .description("show version information") - .handler([&]() { printVersion(programName); }); - - mkFlag() - .longName("no-net") - .description("disable substituters and consider all previously downloaded files up-to-date") - .handler([&]() { useNet = false; }); + }}, + }); + + addFlag({ + .longName = "print-build-logs", + .shortName = 'L', + .description = "print full build logs on stderr", + .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, + }); + + addFlag({ + .longName = "version", + .description = "show version information", + .handler = {[&]() { printVersion(programName); }}, + }); + + addFlag({ + .longName = "no-net", + .description = "disable substituters and consider all previously downloaded files up-to-date", + .handler = {[&]() { useNet = false; }}, + }); + + addFlag({ + .longName = "refresh", + .description = "consider all previously downloaded files out-of-date", + .handler = {[&]() { refresh = true; }}, + }); + + deprecatedAliases.insert({"dev-shell", "develop"}); } void printFlags(std::ostream & out) override @@ -99,8 +120,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs Args::printFlags(out); std::cout << "\n" - "In addition, most configuration settings can be overriden using '--<name> <value>'.\n" - "Boolean settings can be overriden using '--<name>' or '--no-<name>'. See 'nix\n" + "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" + "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" "--help-config' for a list of configuration settings.\n"; } @@ -109,10 +130,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs MultiCommand::printHelp(programName, out); #if 0 - out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n"; + out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; #endif - std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; } void showHelpAndExit() @@ -145,19 +166,22 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlWarn; settings.verboseBuild = false; + setLogFormat("bar"); + + Finally f([] { logger->stop(); }); + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); - settings.requireExperimentalFeature("nix-command"); - initPlugins(); if (!args.command) args.showHelpAndExit(); - Finally f([]() { stopProgressBar(); }); - - startProgressBar(args.printBuildLogs); + if (args.command->first != "repl" + && args.command->first != "doctor" + && args.command->first != "upgrade-nix") + settings.requireExperimentalFeature("nix-command"); if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); @@ -170,14 +194,17 @@ void mainWrapped(int argc, char * * argv) settings.useSubstitutes = false; if (!settings.tarballTtl.overriden) settings.tarballTtl = std::numeric_limits<unsigned int>::max(); - if (!downloadSettings.tries.overriden) - downloadSettings.tries = 0; - if (!downloadSettings.connectTimeout.overriden) - downloadSettings.connectTimeout = 1; + if (!fileTransferSettings.tries.overriden) + fileTransferSettings.tries = 0; + if (!fileTransferSettings.connectTimeout.overriden) + fileTransferSettings.connectTimeout = 1; } - args.command->prepare(); - args.command->run(); + if (args.refresh) + settings.tarballTtl = 0; + + args.command->second->prepare(); + args.command->second->run(); } } diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index f9c7fef3f..0ebb8f13b 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -31,9 +31,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON }, }; } + + Category category() override { return catUtility; } + void run(ref<Store> store, StorePaths storePaths) override { - auto paths = store->topoSortPaths(storePathsToSet(storePaths)); + auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); std::reverse(paths.begin(), paths.end()); @@ -45,7 +48,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON for (auto & path : paths) { auto pathS = store->printStorePath(path); auto oldInfo = store->queryPathInfo(path); - auto oldHashPart = storePathToHash(pathS); + std::string oldHashPart(path.hashPart()); StringSink sink; store->narFromPath(path, sink); @@ -59,7 +62,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON hasSelfReference = true; else { auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second.clone() : ref.clone(); + auto replacement = i != remappings.end() ? i->second : ref; // FIXME: warn about unremapped paths? if (replacement != ref) rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); @@ -74,18 +77,18 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path.clone()); + if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(true, info.narHash); + info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); if (!json) - printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); + printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); rsink2((unsigned char *) sink.s->data(), sink.s->size()); rsink2.flush(); }); diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index fed012b04..b45951879 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -23,6 +23,8 @@ struct CmdOptimiseStore : StoreCommand }; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { store->optimiseStore(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index bffa7b356..fb7bacc4c 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -29,6 +29,8 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } + Category category() override { return catSecondary; } + Examples examples() override { return { @@ -88,8 +90,8 @@ struct CmdPathInfo : StorePathsCommand, MixJSON JSONPlaceholder jsonRoot(std::cout); store->pathInfoToJSON(jsonRoot, // FIXME: preserve order? - storePathsToSet(storePaths), - true, showClosureSize, AllowInvalid); + StorePathSet(storePaths.begin(), storePaths.end()), + true, showClosureSize, SRI, AllowInvalid); } else { diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 3a2e542a3..127397a29 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -21,6 +21,8 @@ struct CmdPingStore : StoreCommand }; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { store->connect(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 27727bd25..4bcaaeebf 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -82,40 +82,6 @@ struct NixRepl : gc }; -void printHelp() -{ - std::cout - << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" - << "\n" - << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" - << "\n" - << "Options:\n" - << " --help\n" - << " Prints out a summary of the command syntax and exits.\n" - << "\n" - << " --version\n" - << " Prints out the Nix version number on standard output and exits.\n" - << "\n" - << " -I path\n" - << " Add a path to the Nix expression search path. This option may be given\n" - << " multiple times. See the NIX_PATH environment variable for information on\n" - << " the semantics of the Nix search path. Paths added through -I take\n" - << " precedence over NIX_PATH.\n" - << "\n" - << " paths...\n" - << " A list of paths to files containing Nix expressions which nix-repl will\n" - << " load and add to its scope.\n" - << "\n" - << " A path surrounded in < and > will be looked up in the Nix expression search\n" - << " path, as in the Nix language itself.\n" - << "\n" - << " If an element of paths starts with http:// or https://, it is interpreted\n" - << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" - << " location. The tarball must include a single top-level directory containing\n" - << " at least a file named default.nix.\n"; -} - - string removeWhitespace(string s) { s = chomp(s); @@ -252,12 +218,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) // input without clearing the input so far. continue; } else { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); } } catch (Error & e) { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); } catch (Interrupted & e) { - printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg()); } // We handled the current input fully, so we should clear it @@ -546,7 +512,7 @@ bool NixRepl::processLine(string line) return false; else if (command != "") - throw Error(format("unknown command '%1%'") % command); + throw Error("unknown command '%1%'", command); else { size_t p = line.find('='); @@ -809,6 +775,16 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } + Examples examples() override + { + return { + Example{ + "Display all special commands within the REPL:", + "nix repl\n nix-repl> :?" + } + }; + } + void run(ref<Store> store) override { auto repl = std::make_unique<NixRepl>(searchPath, openStore()); diff --git a/src/nix/run.cc b/src/nix/run.cc index f885c5e49..321ee1d11 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -8,6 +8,7 @@ #include "fs-accessor.hh" #include "progress-bar.hh" #include "affinity.hh" +#include "eval.hh" #if __linux__ #include <sys/mount.h> @@ -19,46 +20,59 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct CmdRun : InstallablesCommand +struct RunCommon : virtual Command { - std::vector<std::string> command = { "bash" }; - StringSet keep, unset; - bool ignoreEnvironment = false; + void runProgram(ref<Store> store, + const std::string & program, + const Strings & args) + { + stopProgressBar(); - CmdRun() + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast<LocalStore>(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; + for (auto & arg : args) helperArgs.push_back(arg); + + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", program); + } +}; + +struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment +{ + std::vector<std::string> command = { getEnv("SHELL").value_or("bash") }; + + CmdShell() { - mkFlag() - .longName("command") - .shortName('c') - .description("command and arguments to be executed; defaults to 'bash'") - .labels({"command", "args"}) - .arity(ArityAny) - .handler([&](std::vector<std::string> ss) { + addFlag({ + .longName = "command", + .shortName = 'c', + .description = "command and arguments to be executed; defaults to '$SHELL'", + .labels = {"command", "args"}, + .handler = {[&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; - }); - - mkFlag() - .longName("ignore-environment") - .shortName('i') - .description("clear the entire environment (except those specified with --keep)") - .set(&ignoreEnvironment, true); - - mkFlag() - .longName("keep") - .shortName('k') - .description("keep specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); }); - - mkFlag() - .longName("unset") - .shortName('u') - .description("unset specified environment variable") - .arity(1) - .labels({"name"}) - .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); }); + }} + }); } std::string description() override @@ -71,19 +85,19 @@ struct CmdRun : InstallablesCommand return { Example{ "To start a shell providing GNU Hello from NixOS 17.03:", - "nix run -f channel:nixos-17.03 hello" + "nix shell -f channel:nixos-17.03 hello" }, Example{ "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix run nixpkgs.youtube-dl" + "nix shell nixpkgs.youtube-dl" }, Example{ "To run GNU Hello:", - "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" + "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" }, Example{ "To run GNU Hello in a chroot store:", - "nix run --store ~/my-nix nixpkgs.hello -c hello" + "nix shell --store ~/my-nix nixpkgs.hello -c hello" }, }; } @@ -94,41 +108,19 @@ struct CmdRun : InstallablesCommand auto accessor = store->getFSAccessor(); - if (ignoreEnvironment) { - - if (!unset.empty()) - throw UsageError("--unset does not make sense with --ignore-environment"); - - std::map<std::string, std::string> kept; - for (auto & var : keep) { - auto s = getenv(var.c_str()); - if (s) kept[var] = s; - } - - clearEnv(); - - for (auto & var : kept) - setenv(var.first.c_str(), var.second.c_str(), 1); - - } else { - - if (!keep.empty()) - throw UsageError("--keep does not make sense without --ignore-environment"); - - for (auto & var : unset) - unsetenv(var.c_str()); - } std::unordered_set<StorePath> done; std::queue<StorePath> todo; - for (auto & path : outPaths) todo.push(path.clone()); + for (auto & path : outPaths) todo.push(path); + + setEnviron(); auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":"); while (!todo.empty()) { - auto path = todo.front().clone(); + auto path = todo.front(); todo.pop(); - if (!done.insert(path.clone()).second) continue; + if (!done.insert(path).second) continue; if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); @@ -142,42 +134,14 @@ struct CmdRun : InstallablesCommand setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); - std::string cmd = *command.begin(); Strings args; for (auto & arg : command) args.push_back(arg); - stopProgressBar(); - - restoreSignals(); - - restoreAffinity(); - - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); - - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd }; - for (auto & arg : args) helperArgs.push_back(arg); - - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - - throw SysError("could not execute chroot helper"); - } - - execvp(cmd.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("unable to exec '%s'", cmd); + runProgram(store, *command.begin(), args); } }; -static auto r1 = registerCommand<CmdRun>("run"); +static auto r1 = registerCommand<CmdShell>("shell"); void chrootHelper(int argc, char * * argv) { @@ -233,10 +197,10 @@ void chrootHelper(int argc, char * * argv) Finally freeCwd([&]() { free(cwd); }); if (chroot(tmpDir.c_str()) == -1) - throw SysError(format("chrooting into '%s'") % tmpDir); + throw SysError("chrooting into '%s'", tmpDir); if (chdir(cwd) == -1) - throw SysError(format("chdir to '%s' in chroot") % cwd); + throw SysError("chdir to '%s' in chroot", cwd); } else if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); diff --git a/src/nix/search.cc b/src/nix/search.cc index 769274543..ba72c1e79 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -40,16 +40,18 @@ struct CmdSearch : SourceExprCommand, MixJSON { expectArgs("regex", &res); - mkFlag() - .longName("update-cache") - .shortName('u') - .description("update the package search cache") - .handler([&]() { writeCache = true; useCache = false; }); - - mkFlag() - .longName("no-cache") - .description("do not use or update the package search cache") - .handler([&]() { writeCache = false; useCache = false; }); + addFlag({ + .longName = "update-cache", + .shortName = 'u', + .description = "update the package search cache", + .handler = {[&]() { writeCache = true; useCache = false; }} + }); + + addFlag({ + .longName = "no-cache", + .description = "do not use or update the package search cache", + .handler = {[&]() { writeCache = false; useCache = false; }} + }); } std::string description() override @@ -263,7 +265,7 @@ struct CmdSearch : SourceExprCommand, MixJSON throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); } - if (results.size() == 0) + if (!json && results.size() == 0) throw Error("no results for the given search term(s)!"); RunPager pager; diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 87544f937..4fd8886de 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -13,6 +13,8 @@ struct CmdShowConfig : Command, MixJSON return "show the Nix configuration"; } + Category category() override { return catUtility; } + void run() override { if (json) { @@ -23,7 +25,7 @@ struct CmdShowConfig : Command, MixJSON std::map<std::string, Config::SettingInfo> settings; globalConfig.getSettings(settings); for (auto & s : settings) - std::cout << s.first + " = " + s.second.value + "\n"; + logger->stdout("%s = %s", s.first, s.second.value); } } }; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 0ede7b468..2d31894c2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -15,11 +15,12 @@ struct CmdShowDerivation : InstallablesCommand CmdShowDerivation() { - mkFlag() - .longName("recursive") - .shortName('r') - .description("include the dependencies of the specified derivations") - .set(&recursive, true); + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "include the dependencies of the specified derivations", + .handler = {&recursive, true} + }); } std::string description() override @@ -41,6 +42,8 @@ struct CmdShowDerivation : InstallablesCommand }; } + Category category() override { return catUtility; } + void run(ref<Store> store) override { auto drvPaths = toDerivations(store, installables, true); @@ -58,11 +61,9 @@ struct CmdShowDerivation : InstallablesCommand for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - auto drvPathS = store->printStorePath(drvPath); - - auto drvObj(jsonRoot.object(drvPathS)); + auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); - auto drv = readDerivation(*store, drvPathS); + auto drv = store->readDerivation(drvPath); { auto outputsObj(drvObj.object("outputs")); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 5f07448e0..6c9b9a792 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -13,13 +13,13 @@ struct CmdCopySigs : StorePathsCommand CmdCopySigs() { - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }}, + }); } std::string description() override @@ -27,6 +27,8 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } + Category category() override { return catUtility; } + void run(ref<Store> store, StorePaths storePaths) override { if (substituterUris.empty()) @@ -98,12 +100,13 @@ struct CmdSignPaths : StorePathsCommand CmdSignPaths() { - mkFlag() - .shortName('k') - .longName("key-file") - .label("file") - .description("file containing the secret signing key") - .dest(&secretKeyFile); + addFlag({ + .longName = "key-file", + .shortName = 'k', + .description = "file containing the secret signing key", + .labels = {"file"}, + .handler = {&secretKeyFile} + }); } std::string description() override @@ -111,6 +114,8 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } + Category category() override { return catUtility; } + void run(ref<Store> store, StorePaths storePaths) override { if (secretKeyFile.empty()) diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 87f1f9d1b..a880bdae0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "store-api.hh" -#include "download.hh" +#include "filetransfer.hh" #include "eval.hh" #include "attr-path.hh" #include "names.hh" @@ -16,18 +16,20 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand CmdUpgradeNix() { - mkFlag() - .longName("profile") - .shortName('p') - .labels({"profile-dir"}) - .description("the Nix profile to upgrade") - .dest(&profileDir); - - mkFlag() - .longName("nix-store-paths-url") - .labels({"url"}) - .description("URL of the file that contains the store paths of the latest Nix release") - .dest(&storePathsUrl); + addFlag({ + .longName = "profile", + .shortName = 'p', + .description = "the Nix profile to upgrade", + .labels = {"profile-dir"}, + .handler = {&profileDir} + }); + + addFlag({ + .longName = "nix-store-paths-url", + .description = "URL of the file that contains the store paths of the latest Nix release", + .labels = {"url"}, + .handler = {&storePathsUrl} + }); } std::string description() override @@ -49,6 +51,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }; } + Category category() override { return catNixInstallation; } + void run(ref<Store> store) override { evalSettings.pureEval = true; @@ -64,7 +68,10 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - printError("would upgrade to version %s", version); + logWarning({ + .name = "Version update", + .hint = hintfmt("would upgrade to version %s", version) + }); return; } @@ -90,7 +97,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } - printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); + printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); } /* Return the profile in which Nix is installed. */ @@ -138,14 +145,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = DownloadRequest(storePathsUrl); - auto res = getDownloader()->download(req); + auto req = FileTransferRequest(storePathsUrl); + auto res = getFileTransfer()->download(req); auto state = std::make_unique<EvalState>(Strings(), store); auto v = state->allocValue(); state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); Bindings & bindings(*state->allocBindings(0)); - auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; return store->parseStorePath(state->forceString(*v2)); } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 9b0658803..ab83637dc 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -20,13 +20,13 @@ struct CmdVerify : StorePathsCommand { mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); - mkFlag() - .longName("substituter") - .shortName('s') - .labels({"store-uri"}) - .description("use signatures from specified store") - .arity(1) - .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); }); + addFlag({ + .longName = "substituter", + .shortName = 's', + .description = "use signatures from specified store", + .labels = {"store-uri"}, + .handler = {[&](std::string s) { substituterUris.push_back(s); }} + }); mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); } @@ -49,6 +49,8 @@ struct CmdVerify : StorePathsCommand }; } + Category category() override { return catSecondary; } + void run(ref<Store> store, StorePaths storePaths) override { std::vector<ref<Store>> substituters; @@ -88,7 +90,7 @@ struct CmdVerify : StorePathsCommand if (info->ca == "") hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(store->printStorePath(info->path))); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); store->narFromPath(info->path, *hashSink); @@ -97,11 +99,15 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - printError( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string()); + logError({ + .name = "Hash error - path modified", + .hint = hintfmt( + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(Base32, true), + hash.first.to_string(Base32, true)) + }); } - } if (!noTrust) { @@ -137,7 +143,7 @@ struct CmdVerify : StorePathsCommand doSigs(info2->sigs); } catch (InvalidPath &) { } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logError(e.info()); } } @@ -148,7 +154,12 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - printError("path '%s' is untrusted", store->printStorePath(info->path)); + logError({ + .name = "Untrusted path", + .hint = hintfmt("path '%s' is untrusted", + store->printStorePath(info->path)) + }); + } } @@ -156,7 +167,7 @@ struct CmdVerify : StorePathsCommand done++; } catch (Error & e) { - printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logError(e.info()); failed++; } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index c24ae7c8e..167c974ee 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -37,11 +37,12 @@ struct CmdWhyDepends : SourceExprCommand expectArg("package", &_package); expectArg("dependency", &_dependency); - mkFlag() - .longName("all") - .shortName('a') - .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path") - .set(&all, true); + addFlag({ + .longName = "all", + .shortName = 'a', + .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .handler = {&all, true}, + }); } std::string description() override @@ -67,13 +68,15 @@ struct CmdWhyDepends : SourceExprCommand }; } + Category category() override { return catSecondary; } + void run(ref<Store> store) override { auto package = parseInstallable(*this, store, _package, false); auto packagePath = toStorePath(store, Build, package); auto dependency = parseInstallable(*this, store, _dependency, false); auto dependencyPath = toStorePath(store, NoBuild, dependency); - auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath)); + auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); @@ -103,16 +106,16 @@ struct CmdWhyDepends : SourceExprCommand std::map<StorePath, Node> graph; for (auto & path : closure) - graph.emplace(path.clone(), Node { .path = path.clone(), .refs = cloneStorePathSet(store->queryPathInfo(path)->references) }); + graph.emplace(path, Node { .path = path, .refs = store->queryPathInfo(path)->references }); // Transpose the graph. for (auto & node : graph) for (auto & ref : node.second.refs) - graph.find(ref)->second.rrefs.insert(node.first.clone()); + graph.find(ref)->second.rrefs.insert(node.first); /* Run Dijkstra's shortest path algorithm to get the distance of every path in the closure to 'dependency'. */ - graph.emplace(dependencyPath.clone(), Node { .path = dependencyPath.clone(), .dist = 0 }); + graph.emplace(dependencyPath, Node { .path = dependencyPath, .dist = 0 }); std::priority_queue<Node *> queue; @@ -143,21 +146,16 @@ struct CmdWhyDepends : SourceExprCommand and `dependency`. */ std::function<void(Node &, const string &, const string &)> printNode; - const string treeConn = "╠═══"; - const string treeLast = "╚═══"; - const string treeLine = "║ "; - const string treeNull = " "; - struct BailOut { }; printNode = [&](Node & node, const string & firstPad, const string & tailPad) { auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", + logger->stdout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", - firstPad != "" ? "=> " : "", + firstPad != "" ? "→ " : "", pathS); if (node.path == dependencyPath && !all @@ -177,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand auto & node2 = graph.at(ref); if (node2.dist == inf) continue; refs.emplace(node2.dist, &node2); - hashes.insert(storePathToHash(store->printStorePath(node2.path))); + hashes.insert(std::string(node2.path.hashPart())); } /* For each reference, find the files and symlinks that @@ -213,7 +211,7 @@ struct CmdWhyDepends : SourceExprCommand p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), - pos - pos2, storePathHashLen, + pos - pos2, StorePath::HashLen, getColour(hash)))); } } @@ -226,7 +224,7 @@ struct CmdWhyDepends : SourceExprCommand auto pos = target.find(hash); if (pos != std::string::npos) hits[hash].emplace_back(fmt("%s -> %s\n", p2, - hilite(target, pos, storePathHashLen, getColour(hash)))); + hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } }; @@ -237,7 +235,7 @@ struct CmdWhyDepends : SourceExprCommand RunPager pager; for (auto & ref : refs) { - auto hash = storePathToHash(store->printStorePath(ref.second->path)); + std::string hash(ref.second->path.hashPart()); bool last = all ? ref == *refs.rbegin() : true; diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index bf65f7905..054ae01cb 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,8 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix -resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust +resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain + +resolve-system-dependencies_LIBS := libstore libmain libutil resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 8f0c99c84..434ad80a6 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -39,12 +39,18 @@ std::set<std::string> runResolver(const Path & filename) throw SysError("statting '%s'", filename); if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular file", filename); + logError({ + .name = "Regular MACH file", + .hint = hintfmt("file '%s' is not a regular file", filename) + }); return {}; } if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); + logError({ + .name = "File too short", + .hint = hintfmt("file '%s' is too short for a MACH binary", filename) + }); return {}; } @@ -66,13 +72,19 @@ std::set<std::string> runResolver(const Path & filename) } } if (mach64_offset == 0) { - printError(format("Could not find any mach64 blobs in file '%1%', continuing...") % filename); + logError({ + .name = "No mach64 blobs", + .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename) + }); return {}; } } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { mach64_offset = 0; } else { - printError(format("Object file has unknown magic number '%1%', skipping it...") % magic); + logError({ + .name = "Magic number", + .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic) + }); return {}; } diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index eb58ae7c1..17b63d978 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -48,7 +48,7 @@ basicTests # Test HttpBinaryCacheStore. -export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +export _NIX_FORCE_HTTP=1 basicTests @@ -105,10 +105,24 @@ mv $cacheDir/nar2 $cacheDir/nar # incomplete closure. clearStore -rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grep -q "copying path" $TEST_ROOT/log +grep -q "copying path.*input-0" $TEST_ROOT/log +grep -q "copying path.*input-2" $TEST_ROOT/log +grep -q "copying path.*top" $TEST_ROOT/log + + +# Idem, but without cached .narinfo. +clearStore +clearCacheCache + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grep -q "don't know how to build" $TEST_ROOT/log +grep -q "building.*input-1" $TEST_ROOT/log +grep -q "building.*input-2" $TEST_ROOT/log +grep -q "copying path.*input-0" $TEST_ROOT/log +grep -q "copying path.*top" $TEST_ROOT/log if [ -n "$HAVE_SODIUM" ]; then @@ -126,7 +140,7 @@ badKey="$(cat $TEST_ROOT/pk2)" res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) otherKey="$(cat $TEST_ROOT/pk3)" -_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath # Downloading should fail if we don't provide a key. diff --git a/tests/build-hook.nix b/tests/build-hook.nix index 8bff0fe79..8c5ca8cd3 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -4,13 +4,13 @@ let input1 = mkDerivation { name = "build-hook-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { name = "build-hook-input-2"; - builder = ./dependencies.builder2.sh; + buildCommand = "mkdir $out; echo BAR > $out/bar"; }; in diff --git a/tests/build-remote.sh b/tests/build-remote.sh index ddd68f327..a550f4460 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -20,5 +20,5 @@ cat $outPath/foobar | grep FOOBAR # Ensure that input1 was built on store1 due to the required feature. p=$(readlink -f $outPath/input-2) -(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh) -nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh +(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh) +nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh diff --git a/tests/check.nix b/tests/check.nix index 56c82e565..bca04fdaf 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -1,12 +1,45 @@ +{checkBuildId ? 0}: + with import ./config.nix; { nondeterministic = mkDerivation { + inherit checkBuildId; name = "nondeterministic"; buildCommand = '' mkdir $out date +%s.%N > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + deterministic = mkDerivation { + inherit checkBuildId; + name = "deterministic"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + ''; + }; + + failed = mkDerivation { + inherit checkBuildId; + name = "failed"; + buildCommand = + '' + mkdir $out + echo date > $out/date + echo "CHECK_TMPDIR=$TMPDIR" + echo "checkBuildId=$checkBuildId" + echo "$checkBuildId" > $TMPDIR/checkBuildId + false ''; }; diff --git a/tests/check.sh b/tests/check.sh index bc23a6634..5f25d04cb 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -1,14 +1,57 @@ source common.sh +checkBuildTempDirRemoved () +{ + buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1) + checkBuildIdFile=${buildDir}/checkBuildId + [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile +} + +# written to build temp directories to verify created by this instance +checkBuildId=$(date +%s%N) + clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check -nix-build check.nix -A nondeterministic --no-out-link -nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? +# check for dangling temporary build directories +# only retain if build fails and --keep-failed is specified, or... +# ...build is non-deterministic and --check and --keep-failed are both specified +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$? +[ "$status" = "100" ] +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log +if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link 2> $TEST_ROOT/log +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] +checkBuildTempDirRemoved $TEST_ROOT/log + +nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ + --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] +if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi clearStore diff --git a/tests/common.sh.in b/tests/common.sh.in index 15d7b1ef9..dd7e61822 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var export NIX_LOG_DIR=$TEST_ROOT/var/log/nix export NIX_STATE_DIR=$TEST_ROOT/var/nix export NIX_CONF_DIR=$TEST_ROOT/etc +unset NIX_USER_CONF_FILES export _NIX_TEST_SHARED=$TEST_ROOT/shared if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 @@ -21,6 +22,8 @@ export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME mkdir -p $TEST_HOME diff --git a/tests/config.nix.in b/tests/config.nix.in index 51aed539c..a57a8c596 100644 --- a/tests/config.nix.in +++ b/tests/config.nix.in @@ -3,7 +3,7 @@ rec { path = "@coreutils@"; - system = builtins.currentSystem; + system = "@system@"; shared = builtins.getEnv "_NIX_TEST_SHARED"; @@ -11,7 +11,7 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; PATH = path; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/config.sh b/tests/config.sh new file mode 100644 index 000000000..8fa349f11 --- /dev/null +++ b/tests/config.sh @@ -0,0 +1,18 @@ +source common.sh + +# Test that files are loaded from XDG by default +export XDG_CONFIG_HOME=/tmp/home +export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2 +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]] + +# Test that setting NIX_USER_CONF_FILES overrides all the default user config files +export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf +files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs) +[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]] + +# Test that it's possible to load the config from a custom location +here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") +export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf +var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +[[ $var == https://example.com ]] diff --git a/tests/config/nix-with-substituters.conf b/tests/config/nix-with-substituters.conf new file mode 100644 index 000000000..90f359a6f --- /dev/null +++ b/tests/config/nix-with-substituters.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +substituters = https://example.com diff --git a/tests/dependencies.builder1.sh b/tests/dependencies.builder1.sh deleted file mode 100644 index 4b006a17d..000000000 --- a/tests/dependencies.builder1.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir $out -echo FOO > $out/foo diff --git a/tests/dependencies.builder2.sh b/tests/dependencies.builder2.sh deleted file mode 100644 index 4f886fdb3..000000000 --- a/tests/dependencies.builder2.sh +++ /dev/null @@ -1,2 +0,0 @@ -mkdir $out -echo BAR > $out/bar diff --git a/tests/dependencies.nix b/tests/dependencies.nix index eca4b2964..e320d81c9 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -2,18 +2,27 @@ with import ./config.nix; let { + input0 = mkDerivation { + name = "dependencies-input-0"; + buildCommand = "mkdir $out; echo foo > $out/bar"; + }; + input1 = mkDerivation { name = "dependencies-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; }; input2 = mkDerivation { name = "dependencies-input-2"; - builder = "${./dependencies.builder2.sh}"; + buildCommand = '' + mkdir $out + echo BAR > $out/bar + echo ${input0} > $out/input0 + ''; }; body = mkDerivation { - name = "dependencies"; + name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; diff --git a/tests/dependencies.sh b/tests/dependencies.sh index df204d185..092950aa7 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix) echo "derivation is $drvPath" -nix-store -q --tree "$drvPath" | grep ' +---.*builder1.sh' +nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh' # Test Graphviz graph generation. nix-store -q --graph "$drvPath" > $TEST_ROOT/graph @@ -22,9 +22,9 @@ nix-store -q --graph "$outPath" > $TEST_ROOT/graph if test -n "$dot"; then # Does it parse? $dot < $TEST_ROOT/graph -fi +fi -nix-store -q --tree "$outPath" | grep '+---.*dependencies-input-2' +nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" @@ -49,4 +49,4 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" +nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" diff --git a/tests/export-graph.sh b/tests/export-graph.sh index a6fd69054..a1449b34e 100644 --- a/tests/export-graph.sh +++ b/tests/export-graph.sh @@ -11,7 +11,7 @@ checkRef() { outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result) -test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 2 || fail "bad nr of references" +test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 3 || fail "bad nr of references" checkRef input-2 for i in $(cat $outPath); do checkRef $i; done diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index 4c46bdf04..d9c9874f5 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -9,7 +9,9 @@ clearStore repo=$TEST_ROOT/git -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 +export _NIX_FORCE_HTTP=1 + +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow git init $repo git -C $repo config user.email "foobar@example.com" @@ -23,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD) echo world > $repo/hello git -C $repo commit -m 'Bla2' -a +git -C $repo worktree add $TEST_ROOT/worktree +echo hello >> $TEST_ROOT/worktree/hello rev2=$(git -C $repo rev-parse HEAD) +# Fetch a worktree +unset _NIX_FORCE_HTTP +path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") +export _NIX_FORCE_HTTP=1 +[[ $(tail -n 1 $path0/hello) = "hello" ]] + # Fetch the default branch. path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(cat $path/hello) = world ]] @@ -48,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") [[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] [[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] -# But with TTL 0, it should fail. -(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) - # Fetching with a explicit hash should succeed. path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") [[ $path = $path2 ]] @@ -72,6 +79,7 @@ echo bar > $repo/dir2/bar git -C $repo add dir1/foo git -C $repo rm hello +unset _NIX_FORCE_HTTP path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") [ ! -e $path2/hello ] [ ! -e $path2/bar ] @@ -108,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") git -C $repo checkout $rev2 -b dev echo dev > $repo/hello -# File URI uses 'master' unless specified otherwise +# File URI uses dirty tree unless specified otherwise path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") -[[ $path = $path2 ]] +[ $(cat $path2/hello) = dev ] # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") @@ -129,9 +137,9 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP # Nuke the cache -rm -rf $TEST_HOME/.cache/nix/gitv2 +rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH +# Try again, but without 'git' on PATH. This should fail. NIX=$(command -v nix) # This should fail (! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) @@ -139,3 +147,13 @@ NIX=$(command -v nix) # Try again, with 'git' available. This should work. path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] + +# Fetching a shallow repo shouldn't work by default, because we can't +# return a revCount. +git clone --depth 1 file://$repo $TEST_ROOT/shallow +(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath") + +# But you can request a shallow clone, which won't return a revCount. +path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath") +[[ $path3 = $path6 ]] +[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh new file mode 100644 index 000000000..23934698e --- /dev/null +++ b/tests/fetchGitRefs.sh @@ -0,0 +1,111 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git tests" + exit 99 +fi + +clearStore + +repo="$TEST_ROOT/git" + +rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix" + +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +echo utrecht > "$repo"/hello +git -C "$repo" add hello +git -C "$repo" commit -m 'Bla1' + +path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") + +# Test various combinations of ref names +# (taken from the git project) + +# git help check-ref-format +# Git imposes the following rules on how references are named: +# +# 1. They can include slash / for hierarchical (directory) grouping, but no slash-separated component can begin with a dot . or end with the sequence .lock. +# 2. They must contain at least one /. This enforces the presence of a category like heads/, tags/ etc. but the actual names are not restricted. If the --allow-onelevel option is used, this rule is waived. +# 3. They cannot have two consecutive dots .. anywhere. +# 4. They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere. +# 5. They cannot have question-mark ?, asterisk *, or open bracket [ anywhere. See the --refspec-pattern option below for an exception to this rule. +# 6. They cannot begin or end with a slash / or contain multiple consecutive slashes (see the --normalize option below for an exception to this rule) +# 7. They cannot end with a dot .. +# 8. They cannot contain a sequence @{. +# 9. They cannot be the single character @. +# 10. They cannot contain a \. + +valid_ref() { + { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + git check-ref-format --branch "$1" >/dev/null + git -C "$repo" branch "$1" master >/dev/null + path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") + [[ $path1 = $path ]] + git -C "$repo" branch -D "$1" >/dev/null +} + +invalid_ref() { + { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + # special case for a sole @: + # --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel + if [ "$1" = "@" ]; then + (! git check-ref-format --allow-onelevel "$1" >/dev/null 2>&1) + else + (! git check-ref-format --branch "$1" >/dev/null 2>&1) + fi + nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null +} + + +valid_ref 'foox' +valid_ref '1337' +valid_ref 'foo.baz' +valid_ref 'foo/bar/baz' +valid_ref 'foo./bar' +valid_ref 'heads/foo@bar' +valid_ref "$(printf 'heads/fu\303\237')" +valid_ref 'foo-bar-baz' +valid_ref '$1' +valid_ref 'foo.locke' + +invalid_ref 'refs///heads/foo' +invalid_ref 'heads/foo/' +invalid_ref '///heads/foo' +invalid_ref '.foo' +invalid_ref './foo' +invalid_ref './foo/bar' +invalid_ref 'foo/./bar' +invalid_ref 'foo/bar/.' +invalid_ref 'foo bar' +invalid_ref 'foo?bar' +invalid_ref 'foo^bar' +invalid_ref 'foo~bar' +invalid_ref 'foo:bar' +invalid_ref 'foo[bar' +invalid_ref 'foo/bar/.' +invalid_ref '.refs/foo' +invalid_ref 'refs/heads/foo.' +invalid_ref 'heads/foo..bar' +invalid_ref 'heads/foo?bar' +invalid_ref 'heads/foo.lock' +invalid_ref 'heads///foo.lock' +invalid_ref 'foo.lock/bar' +invalid_ref 'foo.lock///bar' +invalid_ref 'heads/v@{ation' +invalid_ref 'heads/foo\.ar' # should fail due to \ +invalid_ref 'heads/foo\bar' # should fail due to \ +invalid_ref "$(printf 'heads/foo\t')" # should fail because it has a TAB +invalid_ref "$(printf 'heads/foo\177')" +invalid_ref '@' + +invalid_ref 'foo/*' +invalid_ref '*/foo' +invalid_ref 'foo/*/bar' +invalid_ref '*' +invalid_ref 'foo/*/*' +invalid_ref '*/foo/*' +invalid_ref '/foo' +invalid_ref '' diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh new file mode 100644 index 000000000..4c2c13f1a --- /dev/null +++ b/tests/fetchGitSubmodules.sh @@ -0,0 +1,97 @@ +source common.sh + +set -u + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git submodule tests" + exit 99 +fi + +clearStore + +rootRepo=$TEST_ROOT/gitSubmodulesRoot +subRepo=$TEST_ROOT/gitSubmodulesSub + +rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix + +initGitRepo() { + git init $1 + git -C $1 config user.email "foobar@example.com" + git -C $1 config user.name "Foobar" +} + +addGitContent() { + echo "lorem ipsum" > $1/content + git -C $1 add content + git -C $1 commit -m "Initial commit" +} + +initGitRepo $subRepo +addGitContent $subRepo + +initGitRepo $rootRepo + +git -C $rootRepo submodule init +git -C $rootRepo submodule add $subRepo sub +git -C $rootRepo add sub +git -C $rootRepo commit -m "Add submodule" + +rev=$(git -C $rootRepo rev-parse HEAD) + +r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") +r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r2 ]] +[[ $r2 != $r3 ]] + +r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") +r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") +r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") +r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") + +[[ $r1 == $r4 ]] +[[ $r4 == $r5 ]] +[[ $r3 == $r6 ]] +[[ $r6 == $r7 ]] +[[ $r7 == $r8 ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules") +[[ $have_submodules == false ]] + +have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules") +[[ $have_submodules == true ]] + +pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") +pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") +pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") + +# The resulting store path cannot be the same. +[[ $pathWithoutSubmodules != $pathWithSubmodules ]] + +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] + +# Checking out the same repo with submodules returns in the same store path. +[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] + +# The submodules flag is actually honored. +[[ ! -e $pathWithoutSubmodules/sub/content ]] +[[ -e $pathWithSubmodules/sub/content ]] + +[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]] + +# No .git directory or submodule reference files must be left +test "$(find "$pathWithSubmodules" -name .git)" = "" + +# Git repos without submodules can be fetched with submodules = true. +subRev=$(git -C $subRepo rev-parse HEAD) +noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") +noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") + +[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix index c0595cc47..21671ea2c 100644 --- a/tests/gc-concurrent.nix +++ b/tests/gc-concurrent.nix @@ -4,12 +4,12 @@ rec { input1 = mkDerivation { name = "dependencies-input-1"; - builder = ./dependencies.builder1.sh; + buildCommand = "mkdir $out; echo FOO > $out/foo"; }; input2 = mkDerivation { name = "dependencies-input-2"; - builder = ./dependencies.builder2.sh; + buildCommand = "mkdir $out; echo BAR > $out/bar"; }; test1 = mkDerivation { @@ -23,5 +23,5 @@ rec { builder = ./gc-concurrent2.builder.sh; inherit input1 input2; }; - + } diff --git a/tests/init.sh b/tests/init.sh index 6a119aad0..c62c4856a 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF build-users-group = keep-derivations = false sandbox = false -experimental-features = nix-command +experimental-features = nix-command flakes include nix.conf.extra EOF diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/lang/eval-okay-getattrpos-functionargs.exp new file mode 100644 index 000000000..7f9ac40e8 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.exp @@ -0,0 +1 @@ +{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; } diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/lang/eval-okay-getattrpos-functionargs.nix new file mode 100644 index 000000000..11d6bb0e3 --- /dev/null +++ b/tests/lang/eval-okay-getattrpos-functionargs.nix @@ -0,0 +1,4 @@ +let + fun = { foo }: {}; + pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun); +in { inherit (pos) column line; file = baseNameOf pos.file; } diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 52967d07d..16abd974c 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store + +# Test that sandboxed builds with --check and -K can move .check directory to store +nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link + +(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log) +if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi +grep -q 'may not be deterministic' $TEST_ROOT/log diff --git a/tests/local.mk b/tests/local.mk index dab3a23b6..536661af8 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,6 @@ nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ + config.sh \ gc.sh \ gc-concurrent.sh \ gc-auto.sh \ @@ -17,9 +18,11 @@ nix_tests = \ nar-access.sh \ structured-attrs.sh \ fetchGit.sh \ + fetchGitRefs.sh \ + fetchGitSubmodules.sh \ fetchMercurial.sh \ signing.sh \ - run.sh \ + shell.sh \ brotli.sh \ pure-eval.sh \ check.sh \ diff --git a/tests/misc.sh b/tests/misc.sh index eda016416..fd4908e25 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -16,4 +16,6 @@ nix-env --foo 2>&1 | grep "no operation" nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. -nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$" +eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) +echo $eval_res | grep "(string) (1:15)" +echo $eval_res | grep "infinite recursion encountered" diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 93f837bef..49c68981a 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -32,10 +32,10 @@ if [ "$xmllint" != false ]; then $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" fi grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. -nix-env -i dependencies +nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] clearProfiles @@ -51,9 +51,9 @@ if [ "$xmllint" != false ]; then $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" fi grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. -nix-env -i dependencies +nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk index 1d2bac052..82ad99402 100644 --- a/tests/plugins/local.mk +++ b/tests/plugins/local.mk @@ -7,3 +7,5 @@ libplugintest_SOURCES := $(d)/plugintest.cc libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 + +libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 77437658e..4cc73465a 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -4,7 +4,7 @@ clearStore startDaemon -storeCleared=1 $SHELL ./user-envs.sh +storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh nix-store --dump-db > $TEST_ROOT/d1 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 diff --git a/tests/run.sh b/tests/run.sh deleted file mode 100644 index d1dbfd6bd..000000000 --- a/tests/run.sh +++ /dev/null @@ -1,28 +0,0 @@ -source common.sh - -clearStore -clearCache - -nix run -f run.nix hello -c hello | grep 'Hello World' -nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS' - -if ! canUseSandbox; then exit; fi - -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 - -clearStore - -path=$(nix eval --raw -f run.nix hello) - -# Note: we need the sandbox paths to ensure that the shell is -# visible in the sandbox. -nix run --sandbox-build-dir /build-tmp \ - --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ - --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World' - -path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello') - -[[ $path/bin/hello = $path2 ]] - -[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/tests/run.nix b/tests/shell-hello.nix index 77dcbd2a9..77dcbd2a9 100644 --- a/tests/run.nix +++ b/tests/shell-hello.nix diff --git a/tests/shell.sh b/tests/shell.sh new file mode 100644 index 000000000..7a9ee8ab0 --- /dev/null +++ b/tests/shell.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore +clearCache + +nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' +nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' + +if ! canUseSandbox; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +clearStore + +path=$(nix eval --raw -f shell-hello.nix hello) + +# Note: we need the sandbox paths to ensure that the shell is +# visible in the sandbox. +nix shell --sandbox-build-dir /build-tmp \ + --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ + --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World' + +path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello') + +[[ $path/bin/hello = $path2 ]] + +[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/tests/tarball.sh b/tests/tarball.sh index 8adb8d72f..b3ec16d40 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -10,6 +10,8 @@ mkdir -p $tarroot cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ +hash=$(nix hash-path $tarroot) + test_tarball() { local ext="$1" local compressor="$2" @@ -25,6 +27,11 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input' + nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext) |