aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.dir-locals.el1
-rw-r--r--.github/dependabot.yml6
-rw-r--r--.github/workflows/test.yml7
-rw-r--r--.version2
-rw-r--r--Makefile1
-rw-r--r--Makefile.config.in1
-rw-r--r--README.md25
-rw-r--r--configure.ac1
-rw-r--r--default.nix3
-rw-r--r--doc/manual/command-ref/conf-file.xml28
-rw-r--r--doc/manual/expressions/builder-syntax.xml119
-rw-r--r--doc/manual/hacking.xml45
-rw-r--r--flake.lock26
-rw-r--r--flake.nix443
-rw-r--r--local.mk2
-rwxr-xr-xmaintainers/upload-release.pl14
-rw-r--r--misc/bash/completion.sh19
-rw-r--r--misc/bash/local.mk1
-rw-r--r--mk/precompiled-headers.mk4
-rwxr-xr-xmk/run_test.sh28
-rw-r--r--mk/tests.mk44
-rw-r--r--perl/lib/Nix/Store.xs11
-rw-r--r--release-common.nix82
-rw-r--r--release.nix303
-rw-r--r--scripts/install-multi-user.sh43
-rw-r--r--scripts/install-nix-from-closure.sh8
-rw-r--r--shell.nix28
-rw-r--r--src/build-remote/build-remote.cc8
-rw-r--r--src/libexpr/attr-path.cc2
-rw-r--r--src/libexpr/attr-set.hh2
-rw-r--r--src/libexpr/common-eval-args.cc25
-rw-r--r--src/libexpr/eval-cache.cc616
-rw-r--r--src/libexpr/eval-cache.hh121
-rw-r--r--src/libexpr/eval-inline.hh4
-rw-r--r--src/libexpr/eval.cc92
-rw-r--r--src/libexpr/eval.hh15
-rw-r--r--src/libexpr/flake/call-flake.nix56
-rw-r--r--src/libexpr/flake/flake.cc609
-rw-r--r--src/libexpr/flake/flake.hh111
-rw-r--r--src/libexpr/flake/flakeref.cc204
-rw-r--r--src/libexpr/flake/flakeref.hh53
-rw-r--r--src/libexpr/flake/lockfile.cc338
-rw-r--r--src/libexpr/flake/lockfile.hh85
-rw-r--r--src/libexpr/get-drvs.cc2
-rw-r--r--src/libexpr/local.mk12
-rw-r--r--src/libexpr/nixexpr.cc19
-rw-r--r--src/libexpr/nixexpr.hh9
-rw-r--r--src/libexpr/parser.y47
-rw-r--r--src/libexpr/primops.cc172
-rw-r--r--src/libexpr/primops/context.cc6
-rw-r--r--src/libexpr/primops/fetchGit.cc91
-rw-r--r--src/libexpr/primops/fetchMercurial.cc18
-rw-r--r--src/libexpr/primops/fetchTree.cc137
-rw-r--r--src/libexpr/primops/fromTOML.cc2
-rw-r--r--src/libexpr/symbol-table.hh11
-rw-r--r--src/libexpr/value.hh7
-rw-r--r--src/libfetchers/attrs.cc14
-rw-r--r--src/libfetchers/attrs.hh11
-rw-r--r--src/libfetchers/cache.hh2
-rw-r--r--src/libfetchers/fetchers.cc258
-rw-r--r--src/libfetchers/fetchers.hh93
-rw-r--r--src/libfetchers/git.cc303
-rw-r--r--src/libfetchers/github.cc304
-rw-r--r--src/libfetchers/indirect.cc104
-rw-r--r--src/libfetchers/mercurial.cc224
-rw-r--r--src/libfetchers/path.cc165
-rw-r--r--src/libfetchers/registry.cc212
-rw-r--r--src/libfetchers/registry.hh64
-rw-r--r--src/libfetchers/tarball.cc142
-rw-r--r--src/libfetchers/tree-info.cc14
-rw-r--r--src/libfetchers/tree-info.hh29
-rw-r--r--src/libmain/common-args.cc12
-rw-r--r--src/libmain/loggers.cc5
-rw-r--r--src/libmain/progress-bar.cc2
-rw-r--r--src/libmain/shared.cc10
-rw-r--r--src/libmain/shared.hh4
-rw-r--r--src/libstore/binary-cache-store.cc132
-rw-r--r--src/libstore/binary-cache-store.hh9
-rw-r--r--src/libstore/build.cc162
-rw-r--r--src/libstore/builtins/buildenv.hh2
-rw-r--r--src/libstore/builtins/fetchurl.cc14
-rw-r--r--src/libstore/content-address.cc16
-rw-r--r--src/libstore/content-address.hh2
-rw-r--r--src/libstore/daemon.cc180
-rw-r--r--src/libstore/derivations.cc126
-rw-r--r--src/libstore/derivations.hh27
-rw-r--r--src/libstore/export-import.cc51
-rw-r--r--src/libstore/filetransfer.cc57
-rw-r--r--src/libstore/filetransfer.hh8
-rw-r--r--src/libstore/gc.cc44
-rw-r--r--src/libstore/globals.cc2
-rw-r--r--src/libstore/globals.hh13
-rw-r--r--src/libstore/http-binary-cache-store.cc4
-rw-r--r--src/libstore/legacy-ssh-store.cc7
-rw-r--r--src/libstore/local-binary-cache-store.cc35
-rw-r--r--src/libstore/local-fs-store.cc6
-rw-r--r--src/libstore/local-store.cc195
-rw-r--r--src/libstore/local-store.hh25
-rw-r--r--src/libstore/local.mk3
-rw-r--r--src/libstore/misc.cc24
-rw-r--r--src/libstore/nar-accessor.cc50
-rw-r--r--src/libstore/nar-accessor.hh4
-rw-r--r--src/libstore/nar-info-disk-cache.cc4
-rw-r--r--src/libstore/nar-info.cc28
-rw-r--r--src/libstore/nar-info.hh4
-rw-r--r--src/libstore/optimise-store.cc2
-rw-r--r--src/libstore/path-info.hh32
-rw-r--r--src/libstore/path.cc2
-rw-r--r--src/libstore/path.hh3
-rw-r--r--src/libstore/profiles.cc81
-rw-r--r--src/libstore/profiles.hh25
-rw-r--r--src/libstore/references.cc21
-rw-r--r--src/libstore/references.hh3
-rw-r--r--src/libstore/remote-fs-accessor.cc33
-rw-r--r--src/libstore/remote-fs-accessor.hh6
-rw-r--r--src/libstore/remote-store.cc162
-rw-r--r--src/libstore/remote-store.hh8
-rw-r--r--src/libstore/s3-binary-cache-store.cc36
-rw-r--r--src/libstore/s3-binary-cache-store.hh1
-rw-r--r--src/libstore/sqlite.cc9
-rw-r--r--src/libstore/sqlite.hh4
-rw-r--r--src/libstore/store-api.cc250
-rw-r--r--src/libstore/store-api.hh89
-rw-r--r--src/libstore/worker-protocol.hh10
-rw-r--r--src/libutil/ansicolor.hh4
-rw-r--r--src/libutil/archive.cc12
-rw-r--r--src/libutil/archive.hh28
-rw-r--r--src/libutil/args.cc139
-rw-r--r--src/libutil/args.hh131
-rw-r--r--src/libutil/error.cc246
-rw-r--r--src/libutil/error.hh93
-rw-r--r--src/libutil/hash.cc109
-rw-r--r--src/libutil/hash.hh24
-rw-r--r--src/libutil/istringstream_nocopy.hh92
-rw-r--r--src/libutil/logging.cc9
-rw-r--r--src/libutil/logging.hh11
-rw-r--r--src/libutil/serialise.cc13
-rw-r--r--src/libutil/serialise.hh75
-rw-r--r--src/libutil/tests/logging.cc133
-rw-r--r--src/libutil/util.cc40
-rw-r--r--src/libutil/util.hh12
-rwxr-xr-xsrc/nix-build/nix-build.cc6
-rw-r--r--src/nix-collect-garbage/nix-collect-garbage.cc6
-rw-r--r--src/nix-env/nix-env.cc35
-rw-r--r--src/nix-prefetch-url/nix-prefetch-url.cc27
-rw-r--r--src/nix-store/nix-store.cc42
-rw-r--r--src/nix/add-to-store.cc21
-rw-r--r--src/nix/app.cc46
-rw-r--r--src/nix/build.cc13
-rw-r--r--src/nix/bundle.cc129
-rw-r--r--src/nix/cat.cc12
-rw-r--r--src/nix/command.cc5
-rw-r--r--src/nix/command.hh119
-rw-r--r--src/nix/copy.cc13
-rw-r--r--src/nix/develop.cc57
-rw-r--r--src/nix/diff-closures.cc146
-rw-r--r--src/nix/edit.cc3
-rw-r--r--src/nix/eval.cc36
-rw-r--r--src/nix/flake.cc980
-rw-r--r--src/nix/get-env.sh9
-rw-r--r--src/nix/hash.cc6
-rw-r--r--src/nix/installables.cc739
-rw-r--r--src/nix/installables.hh88
-rw-r--r--src/nix/log.cc4
-rw-r--r--src/nix/ls.cc12
-rw-r--r--src/nix/main.cc23
-rw-r--r--src/nix/make-content-addressable.cc6
-rw-r--r--src/nix/path-info.cc4
-rw-r--r--src/nix/profile.cc469
-rw-r--r--src/nix/registry.cc151
-rw-r--r--src/nix/repl.cc17
-rw-r--r--src/nix/run.cc74
-rw-r--r--src/nix/search.cc235
-rw-r--r--src/nix/show-derivation.cc10
-rw-r--r--src/nix/sigs.cc3
-rw-r--r--src/nix/verify.cc15
-rw-r--r--src/nix/why-depends.cc26
-rw-r--r--tests/binary-cache.sh53
-rw-r--r--tests/check.sh14
-rw-r--r--tests/common.sh.in5
-rw-r--r--tests/fetchGit.sh67
-rw-r--r--tests/fetchGitRefs.sh6
-rw-r--r--tests/fetchGitSubmodules.sh34
-rw-r--r--tests/fetchMercurial.sh38
-rw-r--r--tests/fetchurl.sh24
-rw-r--r--tests/filter-source.sh18
-rw-r--r--tests/flakes.sh718
-rw-r--r--tests/gc-auto.sh55
-rw-r--r--tests/gc-concurrent.builder.sh5
-rw-r--r--tests/gc-concurrent.nix3
-rw-r--r--tests/gc-concurrent.sh21
-rw-r--r--tests/gc-concurrent2.builder.sh2
-rw-r--r--tests/github-flakes.nix148
-rw-r--r--tests/init.sh2
-rw-r--r--tests/local-store.sh20
-rw-r--r--tests/local.mk9
-rw-r--r--tests/misc.sh11
-rw-r--r--tests/nix-copy-closure.nix9
-rw-r--r--tests/nix-shell.sh7
-rw-r--r--tests/path.nix14
-rw-r--r--tests/plugins.sh2
-rw-r--r--tests/pure-eval.sh16
-rw-r--r--tests/recursive.sh8
-rw-r--r--tests/remote-builds.nix9
-rw-r--r--tests/restricted.sh18
-rw-r--r--tests/search.sh34
-rw-r--r--tests/setuid.nix8
-rw-r--r--tests/shell.nix1
-rw-r--r--tests/tarball.sh11
209 files changed, 10852 insertions, 3448 deletions
diff --git a/.dir-locals.el b/.dir-locals.el
index 8b21d4e40..a2d1dc48d 100644
--- a/.dir-locals.el
+++ b/.dir-locals.el
@@ -1,6 +1,7 @@
((c++-mode . (
(c-file-style . "k&r")
(c-basic-offset . 4)
+ (c-block-comment-prefix . " ")
(indent-tabs-mode . nil)
(tab-width . 4)
(show-trailing-whitespace . t)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..5ace4600a
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,6 @@
+version: 2
+updates:
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 7feefc855..e16e6c62d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,5 +10,8 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- - uses: cachix/install-nix-action@v8
- - run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
+ with:
+ fetch-depth: 0
+ - uses: cachix/install-nix-action@v10
+ #- run: nix flake check
+ - run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
diff --git a/.version b/.version
index 7208c2182..f398a2061 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-2.4 \ No newline at end of file
+3.0 \ No newline at end of file
diff --git a/Makefile b/Makefile
index 332e6e971..f472ca7e5 100644
--- a/Makefile
+++ b/Makefile
@@ -11,6 +11,7 @@ makefiles = \
src/resolve-system-dependencies/local.mk \
scripts/local.mk \
corepkgs/local.mk \
+ misc/bash/local.mk \
misc/systemd/local.mk \
misc/launchd/local.mk \
misc/upstart/local.mk \
diff --git a/Makefile.config.in b/Makefile.config.in
index b632444e8..5c245b8e9 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -19,6 +19,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
OPENSSL_LIBS = @OPENSSL_LIBS@
PACKAGE_NAME = @PACKAGE_NAME@
PACKAGE_VERSION = @PACKAGE_VERSION@
+SHELL = @bash@
SODIUM_LIBS = @SODIUM_LIBS@
SQLITE3_LIBS = @SQLITE3_LIBS@
bash = @bash@
diff --git a/README.md b/README.md
index a1588284d..03c5deb7b 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ for more details.
On Linux and macOS the easiest way to Install Nix is to run the following shell command
(as a user other than root):
-```
+```console
$ curl -L https://nixos.org/nix/install | sh
```
@@ -20,27 +20,8 @@ Information on additional installation methods is available on the [Nix download
## Building And Developing
-### Building Nix
-
-You can build Nix using one of the targets provided by [release.nix](./release.nix):
-
-```
-$ nix-build ./release.nix -A build.aarch64-linux
-$ nix-build ./release.nix -A build.x86_64-darwin
-$ nix-build ./release.nix -A build.i686-linux
-$ nix-build ./release.nix -A build.x86_64-linux
-```
-
-### Development Environment
-
-You can use the provided `shell.nix` to get a working development environment:
-
-```
-$ nix-shell
-$ ./bootstrap.sh
-$ ./configure
-$ make
-```
+See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to
+build nix from source with nix-build or how to get a development environment.
## Additional Resources
diff --git a/configure.ac b/configure.ac
index c3007b4b6..2f29cf864 100644
--- a/configure.ac
+++ b/configure.ac
@@ -123,6 +123,7 @@ AC_PATH_PROG(flex, flex, false)
AC_PATH_PROG(bison, bison, false)
AC_PATH_PROG(dot, dot)
AC_PATH_PROG(lsof, lsof, lsof)
+NEED_PROG(jq, jq)
AC_SUBST(coreutils, [$(dirname $(type -p cat))])
diff --git a/default.nix b/default.nix
new file mode 100644
index 000000000..71d1a80ad
--- /dev/null
+++ b/default.nix
@@ -0,0 +1,3 @@
+(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+ src = ./.;
+}).defaultNix
diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml
index 1fa74a143..9c55526a3 100644
--- a/doc/manual/command-ref/conf-file.xml
+++ b/doc/manual/command-ref/conf-file.xml
@@ -370,34 +370,6 @@ false</literal>.</para>
</varlistentry>
- <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term>
-
- <listitem><para>A list of web servers used by
- <function>builtins.fetchurl</function> to obtain files by
- hash. The default is
- <literal>http://tarballs.nixos.org/</literal>. Given a hash type
- <replaceable>ht</replaceable> and a base-16 hash
- <replaceable>h</replaceable>, Nix will try to download the file
- from
- <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>.
- This allows files to be downloaded even if they have disappeared
- from their original URI. For example, given the default mirror
- <literal>http://tarballs.nixos.org/</literal>, when building the derivation
-
-<programlisting>
-builtins.fetchurl {
- url = "https://example.org/foo-1.2.3.tar.xz";
- sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae";
-}
-</programlisting>
-
- Nix will attempt to download this file from
- <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal>
- first. If it is not available there, if will try the original URI.</para></listitem>
-
- </varlistentry>
-
-
<varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term>
<listitem><para>The maximum number of parallel TCP connections
diff --git a/doc/manual/expressions/builder-syntax.xml b/doc/manual/expressions/builder-syntax.xml
deleted file mode 100644
index e51bade44..000000000
--- a/doc/manual/expressions/builder-syntax.xml
+++ /dev/null
@@ -1,119 +0,0 @@
-<section xmlns="http://docbook.org/ns/docbook"
- xmlns:xlink="http://www.w3.org/1999/xlink"
- xmlns:xi="http://www.w3.org/2001/XInclude"
- version="5.0"
- xml:id='sec-builder-syntax'>
-
-<title>Builder Syntax</title>
-
-<example xml:id='ex-hello-builder'><title>Build script for GNU Hello
-(<filename>builder.sh</filename>)</title>
-<programlisting>
-source $stdenv/setup <co xml:id='ex-hello-builder-co-1' />
-
-PATH=$perl/bin:$PATH <co xml:id='ex-hello-builder-co-2' />
-
-tar xvfz $src <co xml:id='ex-hello-builder-co-3' />
-cd hello-*
-./configure --prefix=$out <co xml:id='ex-hello-builder-co-4' />
-make <co xml:id='ex-hello-builder-co-5' />
-make install</programlisting>
-</example>
-
-<para><xref linkend='ex-hello-builder' /> shows the builder referenced
-from Hello's Nix expression (stored in
-<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>).
-The builder can actually be made a lot shorter by using the
-<emphasis>generic builder</emphasis> functions provided by
-<varname>stdenv</varname>, but here we write out the build steps to
-elucidate what a builder does. It performs the following
-steps:</para>
-
-<calloutlist>
-
- <callout arearefs='ex-hello-builder-co-1'>
-
- <para>When Nix runs a builder, it initially completely clears the
- environment (except for the attributes declared in the
- derivation). For instance, the <envar>PATH</envar> variable is
- empty<footnote><para>Actually, it's initialised to
- <filename>/path-not-set</filename> to prevent Bash from setting it
- to a default value.</para></footnote>. This is done to prevent
- undeclared inputs from being used in the build process. If for
- example the <envar>PATH</envar> contained
- <filename>/usr/bin</filename>, then you might accidentally use
- <filename>/usr/bin/gcc</filename>.</para>
-
- <para>So the first step is to set up the environment. This is
- done by calling the <filename>setup</filename> script of the
- standard environment. The environment variable
- <envar>stdenv</envar> points to the location of the standard
- environment being used. (It wasn't specified explicitly as an
- attribute in <xref linkend='ex-hello-nix' />, but
- <varname>mkDerivation</varname> adds it automatically.)</para>
-
- </callout>
-
- <callout arearefs='ex-hello-builder-co-2'>
-
- <para>Since Hello needs Perl, we have to make sure that Perl is in
- the <envar>PATH</envar>. The <envar>perl</envar> environment
- variable points to the location of the Perl package (since it
- was passed in as an attribute to the derivation), so
- <filename><replaceable>$perl</replaceable>/bin</filename> is the
- directory containing the Perl interpreter.</para>
-
- </callout>
-
- <callout arearefs='ex-hello-builder-co-3'>
-
- <para>Now we have to unpack the sources. The
- <varname>src</varname> attribute was bound to the result of
- fetching the Hello source tarball from the network, so the
- <envar>src</envar> environment variable points to the location in
- the Nix store to which the tarball was downloaded. After
- unpacking, we <command>cd</command> to the resulting source
- directory.</para>
-
- <para>The whole build is performed in a temporary directory
- created in <varname>/tmp</varname>, by the way. This directory is
- removed after the builder finishes, so there is no need to clean
- up the sources afterwards. Also, the temporary directory is
- always newly created, so you don't have to worry about files from
- previous builds interfering with the current build.</para>
-
- </callout>
-
- <callout arearefs='ex-hello-builder-co-4'>
-
- <para>GNU Hello is a typical Autoconf-based package, so we first
- have to run its <filename>configure</filename> script. In Nix
- every package is stored in a separate location in the Nix store,
- for instance
- <filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>.
- Nix computes this path by cryptographically hashing all attributes
- of the derivation. The path is passed to the builder through the
- <envar>out</envar> environment variable. So here we give
- <filename>configure</filename> the parameter
- <literal>--prefix=$out</literal> to cause Hello to be installed in
- the expected location.</para>
-
- </callout>
-
- <callout arearefs='ex-hello-builder-co-5'>
-
- <para>Finally we build Hello (<literal>make</literal>) and install
- it into the location specified by <envar>out</envar>
- (<literal>make install</literal>).</para>
-
- </callout>
-
-</calloutlist>
-
-<para>If you are wondering about the absence of error checking on the
-result of various commands called in the builder: this is because the
-shell script is evaluated with Bash's <option>-e</option> option,
-which causes the script to be aborted if any command fails without an
-error check.</para>
-
-</section> \ No newline at end of file
diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml
index b671811d3..d25d4b84a 100644
--- a/doc/manual/hacking.xml
+++ b/doc/manual/hacking.xml
@@ -4,18 +4,37 @@
<title>Hacking</title>
-<para>This section provides some notes on how to hack on Nix. To get
+<para>This section provides some notes on how to hack on Nix. To get
the latest version of Nix from GitHub:
<screen>
-$ git clone git://github.com/NixOS/nix.git
+$ git clone https://github.com/NixOS/nix.git
$ cd nix
</screen>
</para>
-<para>To build it and its dependencies:
+<para>To build Nix for the current operating system/architecture use
+
+<screen>
+$ nix-build
+</screen>
+
+or if you have a flakes-enabled nix:
+
+<screen>
+$ nix build
+</screen>
+
+This will build <literal>defaultPackage</literal> attribute defined in the <literal>flake.nix</literal> file.
+
+To build for other platforms add one of the following suffixes to it: aarch64-linux,
+i686-linux, x86_64-darwin, x86_64-linux.
+
+i.e.
+
<screen>
-$ nix-build release.nix -A build.x86_64-linux
+nix-build -A defaultPackage.x86_64-linux
</screen>
+
</para>
<para>To build all dependencies and start a shell in which all
@@ -27,13 +46,27 @@ $ nix-shell
To build Nix itself in this shell:
<screen>
[nix-shell]$ ./bootstrap.sh
-[nix-shell]$ configurePhase
-[nix-shell]$ make
+[nix-shell]$ ./configure $configureFlags
+[nix-shell]$ make -j $NIX_BUILD_CORES
</screen>
To install it in <literal>$(pwd)/inst</literal> and test it:
<screen>
[nix-shell]$ make install
[nix-shell]$ make installcheck
+[nix-shell]$ ./inst/bin/nix --version
+nix (Nix) 2.4
+</screen>
+
+If you have a flakes-enabled nix you can replace:
+
+<screen>
+$ nix-shell
+</screen>
+
+by:
+
+<screen>
+$ nix develop
</screen>
</para>
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 000000000..74326d294
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,26 @@
+{
+ "nodes": {
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1591633336,
+ "narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "70717a337f7ae4e486ba71a500367cad697e5f09",
+ "type": "github"
+ },
+ "original": {
+ "id": "nixpkgs",
+ "ref": "nixos-20.03-small",
+ "type": "indirect"
+ }
+ },
+ "root": {
+ "inputs": {
+ "nixpkgs": "nixpkgs"
+ }
+ }
+ },
+ "root": "root",
+ "version": 6
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 000000000..a707e90e7
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,443 @@
+{
+ description = "The purely functional package manager";
+
+ inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
+
+ outputs = { self, nixpkgs }:
+
+ let
+
+ version = builtins.readFile ./.version + versionSuffix;
+ versionSuffix =
+ if officialRelease
+ then ""
+ else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
+
+ officialRelease = false;
+
+ systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
+
+ forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
+
+ # Memoize nixpkgs for different platforms for efficiency.
+ nixpkgsFor = forAllSystems (system:
+ import nixpkgs {
+ inherit system;
+ overlays = [ self.overlay ];
+ }
+ );
+
+ commonDeps = pkgs: with pkgs; rec {
+ # Use "busybox-sandbox-shell" if present,
+ # if not (legacy) fallback and hope it's sufficient.
+ sh = pkgs.busybox-sandbox-shell or (busybox.override {
+ useMusl = true;
+ enableStatic = true;
+ enableMinimal = true;
+ extraConfig = ''
+ CONFIG_FEATURE_FANCY_ECHO y
+ CONFIG_FEATURE_SH_MATH y
+ CONFIG_FEATURE_SH_MATH_64 y
+
+ CONFIG_ASH y
+ CONFIG_ASH_OPTIMIZE_FOR_SIZE y
+
+ CONFIG_ASH_ALIAS y
+ CONFIG_ASH_BASH_COMPAT y
+ CONFIG_ASH_CMDCMD y
+ CONFIG_ASH_ECHO y
+ CONFIG_ASH_GETOPTS y
+ CONFIG_ASH_INTERNAL_GLOB y
+ CONFIG_ASH_JOB_CONTROL y
+ CONFIG_ASH_PRINTF y
+ CONFIG_ASH_TEST y
+ '';
+ });
+
+ configureFlags =
+ lib.optionals stdenv.isLinux [
+ "--with-sandbox-shell=${sh}/bin/busybox"
+ ];
+
+ buildDeps =
+ [ bison
+ flex
+ libxml2
+ libxslt
+ docbook5
+ docbook_xsl_ns
+ autoconf-archive
+ autoreconfHook
+
+ curl
+ bzip2 xz brotli zlib editline
+ openssl pkgconfig sqlite
+ libarchive
+ boost
+ (if lib.versionAtLeast lib.version "20.03pre"
+ then nlohmann_json
+ else nlohmann_json.override { multipleHeaders = true; })
+ nlohmann_json
+
+ # Tests
+ git
+ mercurial
+ jq
+ gmock
+ ]
+ ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
+ ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
+ ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
+ (aws-sdk-cpp.override {
+ apis = ["s3" "transfer"];
+ customMemoryManagement = false;
+ });
+
+ propagatedDeps =
+ [ (boehmgc.override { enableLargeConfig = true; })
+ ];
+
+ perlDeps =
+ [ perl
+ perlPackages.DBDSQLite
+ ];
+ };
+
+ in {
+
+ # A Nixpkgs overlay that overrides the 'nix' and
+ # 'nix.perl-bindings' packages.
+ overlay = final: prev: {
+
+ nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
+ name = "nix-${version}";
+
+ src = self;
+
+ VERSION_SUFFIX = versionSuffix;
+
+ outputs = [ "out" "dev" "doc" ];
+
+ buildInputs = buildDeps;
+
+ propagatedBuildInputs = propagatedDeps;
+
+ preConfigure =
+ ''
+ # Copy libboost_context so we don't get all of Boost in our closure.
+ # https://github.com/NixOS/nixpkgs/issues/45462
+ mkdir -p $out/lib
+ cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
+ rm -f $out/lib/*.a
+ ${lib.optionalString stdenv.isLinux ''
+ chmod u+w $out/lib/*.so.*
+ patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
+ ''}
+ '';
+
+ configureFlags = configureFlags ++
+ [ "--sysconfdir=/etc" ];
+
+ enableParallelBuilding = true;
+
+ makeFlags = "profiledir=$(out)/etc/profile.d";
+
+ doCheck = true;
+
+ installFlags = "sysconfdir=$(out)/etc";
+
+ postInstall = ''
+ mkdir -p $doc/nix-support
+ echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
+ '';
+
+ doInstallCheck = true;
+ installCheckFlags = "sysconfdir=$(out)/etc";
+
+ separateDebugInfo = true;
+ }) // {
+
+ perl-bindings = with final; stdenv.mkDerivation {
+ name = "nix-perl-${version}";
+
+ src = self;
+
+ buildInputs =
+ [ autoconf-archive
+ autoreconfHook
+ nix
+ curl
+ bzip2
+ xz
+ pkgconfig
+ pkgs.perl
+ boost
+ ]
+ ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
+
+ configureFlags = ''
+ --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
+ --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
+ '';
+
+ enableParallelBuilding = true;
+
+ postUnpack = "sourceRoot=$sourceRoot/perl";
+ };
+
+ };
+
+ };
+
+ hydraJobs = {
+
+ # Binary package for various platforms.
+ build = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix);
+
+ # Perl bindings for various platforms.
+ perlBindings = nixpkgs.lib.genAttrs systems (system: nixpkgsFor.${system}.nix.perl-bindings);
+
+ # Binary tarball for various platforms, containing a Nix store
+ # with the closure of 'nix' package, and the second half of
+ # the installation script.
+ binaryTarball = nixpkgs.lib.genAttrs systems (system:
+
+ with nixpkgsFor.${system};
+
+ let
+ installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
+ in
+
+ runCommand "nix-binary-tarball-${version}"
+ { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
+ meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
+ }
+ ''
+ cp ${installerClosureInfo}/registration $TMPDIR/reginfo
+ substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
+ --subst-var-by nix ${nix} \
+ --subst-var-by cacert ${cacert}
+
+ substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
+ --subst-var-by nix ${nix} \
+ --subst-var-by cacert ${cacert}
+ substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
+ --subst-var-by nix ${nix} \
+ --subst-var-by cacert ${cacert}
+ substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
+ --subst-var-by nix ${nix} \
+ --subst-var-by cacert ${cacert}
+
+ if type -p shellcheck; then
+ # SC1090: Don't worry about not being able to find
+ # $nix/etc/profile.d/nix.sh
+ shellcheck --exclude SC1090 $TMPDIR/install
+ shellcheck $TMPDIR/install-darwin-multi-user.sh
+ shellcheck $TMPDIR/install-systemd-multi-user.sh
+
+ # SC1091: Don't panic about not being able to source
+ # /etc/profile
+ # SC2002: Ignore "useless cat" "error", when loading
+ # .reginfo, as the cat is a much cleaner
+ # implementation, even though it is "useless"
+ # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
+ # root's home directory
+ shellcheck --external-sources \
+ --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
+ fi
+
+ chmod +x $TMPDIR/install
+ chmod +x $TMPDIR/install-darwin-multi-user.sh
+ chmod +x $TMPDIR/install-systemd-multi-user.sh
+ chmod +x $TMPDIR/install-multi-user
+ dir=nix-${version}-${system}
+ fn=$out/$dir.tar.xz
+ mkdir -p $out/nix-support
+ echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
+ tar cvfJ $fn \
+ --owner=0 --group=0 --mode=u+rw,uga+r \
+ --absolute-names \
+ --hard-dereference \
+ --transform "s,$TMPDIR/install,$dir/install," \
+ --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
+ --transform "s,$NIX_STORE,$dir/store,S" \
+ $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
+ $TMPDIR/install-systemd-multi-user.sh \
+ $TMPDIR/install-multi-user $TMPDIR/reginfo \
+ $(cat ${installerClosureInfo}/store-paths)
+ '');
+
+ # The first half of the installation script. This is uploaded
+ # to https://nixos.org/nix/install. It downloads the binary
+ # tarball for the user's system and calls the second half of the
+ # installation script.
+ installerScript =
+ with nixpkgsFor.x86_64-linux;
+ runCommand "installer-script"
+ { buildInputs = [ nix ];
+ }
+ ''
+ mkdir -p $out/nix-support
+
+ substitute ${./scripts/install.in} $out/install \
+ ${pkgs.lib.concatMapStrings
+ (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
+ [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
+ } \
+ --replace '@nixVersion@' ${version}
+
+ echo "file installer $out/install" >> $out/nix-support/hydra-build-products
+ '';
+
+ # Line coverage analysis.
+ coverage =
+ with nixpkgsFor.x86_64-linux;
+ with commonDeps pkgs;
+
+ releaseTools.coverageAnalysis {
+ name = "nix-coverage-${version}";
+
+ src = self;
+
+ enableParallelBuilding = true;
+
+ buildInputs = buildDeps ++ propagatedDeps;
+
+ dontInstall = false;
+
+ doInstallCheck = true;
+
+ lcovFilter = [ "*/boost/*" "*-tab.*" ];
+
+ # We call `dot', and even though we just use it to
+ # syntax-check generated dot files, it still requires some
+ # fonts. So provide those.
+ FONTCONFIG_FILE = texFunctions.fontsConf;
+
+ # To test building without precompiled headers.
+ makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
+ };
+
+ # System tests.
+ tests.remoteBuilds = import ./tests/remote-builds.nix {
+ system = "x86_64-linux";
+ inherit nixpkgs;
+ inherit (self) overlay;
+ };
+
+ tests.nix-copy-closure = import ./tests/nix-copy-closure.nix {
+ system = "x86_64-linux";
+ inherit nixpkgs;
+ inherit (self) overlay;
+ };
+
+ tests.githubFlakes = (import ./tests/github-flakes.nix rec {
+ system = "x86_64-linux";
+ inherit nixpkgs;
+ inherit (self) overlay;
+ });
+
+ tests.setuid = nixpkgs.lib.genAttrs
+ ["i686-linux" "x86_64-linux"]
+ (system:
+ import ./tests/setuid.nix rec {
+ inherit nixpkgs system;
+ inherit (self) overlay;
+ });
+
+ # Test whether the binary tarball works in an Ubuntu system.
+ tests.binaryTarball =
+ with nixpkgsFor.x86_64-linux;
+ vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
+ { diskImage = vmTools.diskImages.ubuntu1204x86_64;
+ }
+ ''
+ set -x
+ useradd -m alice
+ su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
+ mkdir /dest-nix
+ mount -o bind /dest-nix /nix # Provide a writable /nix.
+ chown alice /nix
+ su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
+ su - alice -c 'nix-store --verify'
+ su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
+
+ # Check whether 'nix upgrade-nix' works.
+ cat > /tmp/paths.nix <<EOF
+ {
+ x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
+ }
+ EOF
+ su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
+ (! [ -L /home/alice/.profile-1-link ])
+ su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
+
+ mkdir -p $out/nix-support
+ touch $out/nix-support/hydra-build-products
+ umount /nix
+ '');
+
+ /*
+ # Check whether we can still evaluate all of Nixpkgs.
+ tests.evalNixpkgs =
+ import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
+ # FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
+ inherit nixpkgs;
+ pkgs = nixpkgsFor.x86_64-linux;
+ officialRelease = false;
+ };
+
+ # Check whether we can still evaluate NixOS.
+ tests.evalNixOS =
+ with nixpkgsFor.x86_64-linux;
+ runCommand "eval-nixos" { buildInputs = [ nix ]; }
+ ''
+ export NIX_STATE_DIR=$TMPDIR
+
+ nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
+ --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
+
+ touch $out
+ '';
+ */
+
+ };
+
+ checks = forAllSystems (system: {
+ binaryTarball = self.hydraJobs.binaryTarball.${system};
+ perlBindings = self.hydraJobs.perlBindings.${system};
+ });
+
+ packages = forAllSystems (system: {
+ inherit (nixpkgsFor.${system}) nix;
+ });
+
+ defaultPackage = forAllSystems (system: self.packages.${system}.nix);
+
+ devShell = forAllSystems (system:
+ with nixpkgsFor.${system};
+ with commonDeps pkgs;
+
+ stdenv.mkDerivation {
+ name = "nix";
+
+ buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
+
+ inherit configureFlags;
+
+ enableParallelBuilding = true;
+
+ installFlags = "sysconfdir=$(out)/etc";
+
+ shellHook =
+ ''
+ export prefix=$(pwd)/inst
+ configureFlags+=" --prefix=$prefix"
+ PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
+ PATH=$prefix/bin:$PATH
+ unset PYTHONPATH
+ '';
+ });
+
+ };
+}
diff --git a/local.mk b/local.mk
index d254c10fe..b1ca832a6 100644
--- a/local.mk
+++ b/local.mk
@@ -8,7 +8,7 @@ clean-files += Makefile.config
GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
-$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
+$(foreach i, config.h $(wildcard src/lib*/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
$(GCH) $(PCH): src/libutil/util.hh config.h
diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl
index baefe0f12..91ae896d6 100755
--- a/maintainers/upload-release.pl
+++ b/maintainers/upload-release.pl
@@ -170,15 +170,5 @@ $channelsBucket->add_key(
chdir("/home/eelco/Dev/nix-pristine") or die;
system("git remote update origin") == 0 or die;
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
-
-# Update the website.
-my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
-
-system("cd $siteDir && git pull") == 0 or die;
-
-write_file("$siteDir/nix-release.tt",
- "[%-\n" .
- "latestNixVersion = \"$version\"\n" .
- "-%]\n");
-
-system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
+system("git push --tags") == 0 or die;
+system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh
new file mode 100644
index 000000000..bc184edd6
--- /dev/null
+++ b/misc/bash/completion.sh
@@ -0,0 +1,19 @@
+function _complete_nix {
+ local -a words
+ local cword cur
+ _get_comp_words_by_ref -n ':=&' words cword cur
+ local have_type
+ while IFS= read -r line; do
+ if [[ -z $have_type ]]; then
+ have_type=1
+ if [[ $line = filenames ]]; then
+ compopt -o filenames
+ fi
+ else
+ COMPREPLY+=("$line")
+ fi
+ done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
+ __ltrim_colon_completions "$cur"
+}
+
+complete -F _complete_nix nix
diff --git a/misc/bash/local.mk b/misc/bash/local.mk
new file mode 100644
index 000000000..66235af05
--- /dev/null
+++ b/misc/bash/local.mk
@@ -0,0 +1 @@
+$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/nix, 0644))
diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk
index 1c0452dc2..500c99e4a 100644
--- a/mk/precompiled-headers.mk
+++ b/mk/precompiled-headers.mk
@@ -21,13 +21,13 @@ clean-files += $(GCH) $(PCH)
ifeq ($(PRECOMPILE_HEADERS), 1)
- ifeq ($(CXX), g++)
+ ifeq ($(findstring g++,$(CXX)), g++)
GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
GLOBAL_ORDER_AFTER += $(GCH)
- else ifeq ($(CXX), clang++)
+ else ifeq ($(findstring clang++,$(CXX)), clang++)
GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
diff --git a/mk/run_test.sh b/mk/run_test.sh
new file mode 100755
index 000000000..6af5b070a
--- /dev/null
+++ b/mk/run_test.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -u
+
+red=""
+green=""
+yellow=""
+normal=""
+
+post_run_msg="ran test $1..."
+if [ -t 1 ]; then
+ red=""
+ green=""
+ yellow=""
+ normal=""
+fi
+(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
+log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
+status=$?
+if [ $status -eq 0 ]; then
+ echo "$post_run_msg [${green}PASS$normal]"
+elif [ $status -eq 99 ]; then
+ echo "$post_run_msg [${yellow}SKIP$normal]"
+else
+ echo "$post_run_msg [${red}FAIL$normal]"
+ echo "$log" | sed 's/^/ /'
+ exit "$status"
+fi
diff --git a/mk/tests.mk b/mk/tests.mk
index 70c30661b..2e39bb694 100644
--- a/mk/tests.mk
+++ b/mk/tests.mk
@@ -1,45 +1,15 @@
# Run program $1 as part of ‘make installcheck’.
+
+test-deps =
+
define run-install-test
- installcheck: $1
+ installcheck: $1.test
- _installcheck-list += $1
+ .PHONY: $1.test
+ $1.test: $1 $(test-deps)
+ @env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
endef
-# Color code from https://unix.stackexchange.com/a/10065
-installcheck:
- @total=0; failed=0; \
- red=""; \
- green=""; \
- yellow=""; \
- normal=""; \
- if [ -t 1 ]; then \
- red=""; \
- green=""; \
- yellow=""; \
- normal=""; \
- fi; \
- for i in $(_installcheck-list); do \
- total=$$((total + 1)); \
- printf "running test $$i..."; \
- log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
- status=$$?; \
- if [ $$status -eq 0 ]; then \
- echo " [$${green}PASS$$normal]"; \
- elif [ $$status -eq 99 ]; then \
- echo " [$${yellow}SKIP$$normal]"; \
- else \
- echo " [$${red}FAIL$$normal]"; \
- echo "$$log" | sed 's/^/ /'; \
- failed=$$((failed + 1)); \
- fi; \
- done; \
- if [ "$$failed" != 0 ]; then \
- echo "$${red}$$failed out of $$total tests failed $$normal"; \
- exit 1; \
- else \
- echo "$${green}All tests succeeded$$normal"; \
- fi
-
.PHONY: check installcheck
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs
index 945ed49c7..c412e4640 100644
--- a/perl/lib/Nix/Store.xs
+++ b/perl/lib/Nix/Store.xs
@@ -80,7 +80,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
- auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
+ auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef);
else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
- auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
+ auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
@@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
PPCODE:
try {
FdSource source(fd);
- store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
+ store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
} catch (Error & e) {
croak("%s", e.what());
}
@@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath)
HV * outputs = newHV();
for (auto & i : drv.outputs)
- hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0);
+ hv_store(
+ outputs, i.first.c_str(), i.first.size(),
+ newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
+ 0);
hv_stores(hash, "outputs", newRV((SV *) outputs));
AV * inputDrvs = newAV();
diff --git a/release-common.nix b/release-common.nix
deleted file mode 100644
index 4316c3c23..000000000
--- a/release-common.nix
+++ /dev/null
@@ -1,82 +0,0 @@
-{ pkgs }:
-
-with pkgs;
-
-rec {
- # Use "busybox-sandbox-shell" if present,
- # if not (legacy) fallback and hope it's sufficient.
- sh = pkgs.busybox-sandbox-shell or (busybox.override {
- useMusl = true;
- enableStatic = true;
- enableMinimal = true;
- extraConfig = ''
- CONFIG_FEATURE_FANCY_ECHO y
- CONFIG_FEATURE_SH_MATH y
- CONFIG_FEATURE_SH_MATH_64 y
-
- CONFIG_ASH y
- CONFIG_ASH_OPTIMIZE_FOR_SIZE y
-
- CONFIG_ASH_ALIAS y
- CONFIG_ASH_BASH_COMPAT y
- CONFIG_ASH_CMDCMD y
- CONFIG_ASH_ECHO y
- CONFIG_ASH_GETOPTS y
- CONFIG_ASH_INTERNAL_GLOB y
- CONFIG_ASH_JOB_CONTROL y
- CONFIG_ASH_PRINTF y
- CONFIG_ASH_TEST y
- '';
- });
-
- configureFlags =
- lib.optionals stdenv.isLinux [
- "--with-sandbox-shell=${sh}/bin/busybox"
- ];
-
- buildDeps =
- [ bison
- flex
- libxml2
- libxslt
- docbook5
- docbook_xsl_ns
- autoconf-archive
- autoreconfHook
-
- curl
- bzip2 xz brotli zlib editline
- openssl pkgconfig sqlite
- libarchive
- boost
- nlohmann_json
-
- # Tests
- git
- mercurial
- gmock
- ]
- ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
- ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
- ++ lib.optional (stdenv.isLinux || stdenv.isDarwin)
- ((aws-sdk-cpp.override {
- apis = ["s3" "transfer"];
- customMemoryManagement = false;
- }).overrideDerivation (args: {
- /*
- patches = args.patches or [] ++ [ (fetchpatch {
- url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
- sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
- }) ];
- */
- }));
-
- propagatedDeps =
- [ (boehmgc.override { enableLargeConfig = true; })
- ];
-
- perlDeps =
- [ perl
- perlPackages.DBDSQLite
- ];
-}
diff --git a/release.nix b/release.nix
deleted file mode 100644
index fbf9e4721..000000000
--- a/release.nix
+++ /dev/null
@@ -1,303 +0,0 @@
-{ nix ? builtins.fetchGit ./.
-, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
-, officialRelease ? false
-, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
-}:
-
-let
-
- pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
-
- version =
- builtins.readFile ./.version
- + (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
-
- jobs = rec {
-
- build = pkgs.lib.genAttrs systems (system:
-
- let pkgs = import nixpkgs { inherit system; }; in
-
- with pkgs;
-
- with import ./release-common.nix { inherit pkgs; };
-
- stdenv.mkDerivation {
- name = "nix-${version}";
-
- src = nix;
-
- outputs = [ "out" "dev" "doc" ];
-
- buildInputs = buildDeps;
-
- propagatedBuildInputs = propagatedDeps;
-
- preConfigure =
- ''
- # Copy libboost_context so we don't get all of Boost in our closure.
- # https://github.com/NixOS/nixpkgs/issues/45462
- mkdir -p $out/lib
- cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
- rm -f $out/lib/*.a
- ${lib.optionalString stdenv.isLinux ''
- chmod u+w $out/lib/*.so.*
- patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
- ''}
-
- (cd perl; autoreconf --install --force --verbose)
- '';
-
- configureFlags = configureFlags ++
- [ "--sysconfdir=/etc" ];
-
- enableParallelBuilding = true;
-
- makeFlags = "profiledir=$(out)/etc/profile.d";
-
- installFlags = "sysconfdir=$(out)/etc";
-
- postInstall = ''
- mkdir -p $doc/nix-support
- echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
- '';
-
- doCheck = true;
-
- doInstallCheck = true;
- installCheckFlags = "sysconfdir=$(out)/etc";
-
- separateDebugInfo = true;
- });
-
-
- perlBindings = pkgs.lib.genAttrs systems (system:
-
- let pkgs = import nixpkgs { inherit system; }; in with pkgs;
-
- releaseTools.nixBuild {
- name = "nix-perl-${version}";
-
- src = nix;
-
- buildInputs =
- [ autoconf-archive
- autoreconfHook
- jobs.build.${system}
- curl
- bzip2
- xz
- pkgconfig
- pkgs.perl
- boost
- ]
- ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
-
- configureFlags = ''
- --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
- --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
- '';
-
- enableParallelBuilding = true;
-
- postUnpack = "sourceRoot=$sourceRoot/perl";
- });
-
-
- binaryTarball = pkgs.lib.genAttrs systems (system:
-
- with import nixpkgs { inherit system; };
-
- let
- toplevel = builtins.getAttr system jobs.build;
- installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
- in
-
- runCommand "nix-binary-tarball-${version}"
- { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
- meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
- }
- ''
- cp ${installerClosureInfo}/registration $TMPDIR/reginfo
- cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
- substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
- --subst-var-by nix ${toplevel} \
- --subst-var-by cacert ${cacert}
- substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
- --subst-var-by nix ${toplevel} \
- --subst-var-by cacert ${cacert}
- substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
- --subst-var-by nix ${toplevel} \
- --subst-var-by cacert ${cacert}
- substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
- --subst-var-by nix ${toplevel} \
- --subst-var-by cacert ${cacert}
-
- if type -p shellcheck; then
- # SC1090: Don't worry about not being able to find
- # $nix/etc/profile.d/nix.sh
- shellcheck --exclude SC1090 $TMPDIR/install
- shellcheck $TMPDIR/create-darwin-volume.sh
- shellcheck $TMPDIR/install-darwin-multi-user.sh
- shellcheck $TMPDIR/install-systemd-multi-user.sh
-
- # SC1091: Don't panic about not being able to source
- # /etc/profile
- # SC2002: Ignore "useless cat" "error", when loading
- # .reginfo, as the cat is a much cleaner
- # implementation, even though it is "useless"
- # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
- # root's home directory
- shellcheck --external-sources \
- --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
- fi
-
- chmod +x $TMPDIR/install
- chmod +x $TMPDIR/create-darwin-volume.sh
- chmod +x $TMPDIR/install-darwin-multi-user.sh
- chmod +x $TMPDIR/install-systemd-multi-user.sh
- chmod +x $TMPDIR/install-multi-user
- dir=nix-${version}-${system}
- fn=$out/$dir.tar.xz
- mkdir -p $out/nix-support
- echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
- tar cvfJ $fn \
- --owner=0 --group=0 --mode=u+rw,uga+r \
- --absolute-names \
- --hard-dereference \
- --transform "s,$TMPDIR/install,$dir/install," \
- --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
- --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
- --transform "s,$NIX_STORE,$dir/store,S" \
- $TMPDIR/install \
- $TMPDIR/create-darwin-volume.sh \
- $TMPDIR/install-darwin-multi-user.sh \
- $TMPDIR/install-systemd-multi-user.sh \
- $TMPDIR/install-multi-user \
- $TMPDIR/reginfo \
- $(cat ${installerClosureInfo}/store-paths)
- '');
-
-
- coverage =
- with pkgs;
-
- with import ./release-common.nix { inherit pkgs; };
-
- releaseTools.coverageAnalysis {
- name = "nix-coverage-${version}";
-
- src = nix;
-
- enableParallelBuilding = true;
-
- buildInputs = buildDeps ++ propagatedDeps;
-
- dontInstall = false;
-
- doInstallCheck = true;
-
- lcovFilter = [ "*/boost/*" "*-tab.*" ];
-
- # We call `dot', and even though we just use it to
- # syntax-check generated dot files, it still requires some
- # fonts. So provide those.
- FONTCONFIG_FILE = texFunctions.fontsConf;
-
- # To test building without precompiled headers.
- makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
- };
-
-
- # System tests.
- tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
- inherit nixpkgs;
- nix = build.x86_64-linux; system = "x86_64-linux";
- });
-
- tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
- inherit nixpkgs;
- nix = build.x86_64-linux; system = "x86_64-linux";
- });
-
- tests.setuid = pkgs.lib.genAttrs
- ["i686-linux" "x86_64-linux"]
- (system:
- import ./tests/setuid.nix rec {
- inherit nixpkgs;
- nix = build.${system}; inherit system;
- });
-
- tests.binaryTarball =
- with import nixpkgs { system = "x86_64-linux"; };
- vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
- { diskImage = vmTools.diskImages.ubuntu1204x86_64;
- }
- ''
- set -x
- useradd -m alice
- su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
- mkdir /dest-nix
- mount -o bind /dest-nix /nix # Provide a writable /nix.
- chown alice /nix
- su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
- su - alice -c 'nix-store --verify'
- su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
-
- # Check whether 'nix upgrade-nix' works.
- cat > /tmp/paths.nix <<EOF
- {
- x86_64-linux = "${build.x86_64-linux}";
- }
- EOF
- su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
- (! [ -L /home/alice/.profile-1-link ])
- su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
-
- mkdir -p $out/nix-support
- touch $out/nix-support/hydra-build-products
- umount /nix
- ''); # */
-
- /*
- tests.evalNixpkgs =
- import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
- inherit nixpkgs;
- inherit pkgs;
- nix = build.x86_64-linux;
- officialRelease = false;
- };
-
- tests.evalNixOS =
- pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
- ''
- export NIX_STATE_DIR=$TMPDIR
-
- nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
- --arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
-
- touch $out
- '';
- */
-
-
- installerScript =
- pkgs.runCommand "installer-script"
- { buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
- ''
- mkdir -p $out/nix-support
-
- substitute ${./scripts/install.in} $out/install \
- ${pkgs.lib.concatMapStrings
- (system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
- systems
- } \
- --replace '@nixVersion@' ${version}
-
- echo "file installer $out/install" >> $out/nix-support/hydra-build-products
- '';
-
- };
-
-
-in jobs
diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh
index 00c9d540b..e5cc4d7ed 100644
--- a/scripts/install-multi-user.sh
+++ b/scripts/install-multi-user.sh
@@ -37,6 +37,8 @@ readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-
readonly NIX_INSTALLED_NIX="@nix@"
readonly NIX_INSTALLED_CACERT="@cacert@"
+#readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6"
+#readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2"
readonly EXTRACTED_NIX_PATH="$(dirname "$0")"
readonly ROOT_HOME=$(echo ~root)
@@ -69,9 +71,11 @@ uninstall_directions() {
subheader "Uninstalling nix:"
local step=0
- if poly_service_installed_check; then
+ if [ -e /run/systemd/system ] && poly_service_installed_check; then
step=$((step + 1))
poly_service_uninstall_directions "$step"
+ else
+ step=$((step + 1))
fi
for profile_target in "${PROFILE_TARGETS[@]}"; do
@@ -250,20 +254,42 @@ function finish_success {
echo "But fetching the nixpkgs channel failed. (Are you offline?)"
echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"."
fi
- cat <<EOF
+
+ if [ -e /run/systemd/system ]; then
+ cat <<EOF
+
+Before Nix will work in your existing shells, you'll need to close
+them and open them again. Other than that, you should be ready to go.
+
+Try it! Open a new terminal, and type:
+
+ $ nix-shell -p nix-info --run "nix-info -m"
+
+Thank you for using this installer. If you have any feedback, don't
+hesitate:
+
+$(contactme)
+EOF
+ else
+ cat <<EOF
Before Nix will work in your existing shells, you'll need to close
them and open them again. Other than that, you should be ready to go.
Try it! Open a new terminal, and type:
+ $ sudo nix-daemon
$ nix-shell -p nix-info --run "nix-info -m"
+Additionally, you may want to add nix-daemon to your init-system.
+
Thank you for using this installer. If you have any feedback, don't
hesitate:
$(contactme)
EOF
+ fi
+
}
@@ -664,12 +690,8 @@ main() {
# shellcheck source=./install-darwin-multi-user.sh
. "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh"
elif [ "$(uname -s)" = "Linux" ]; then
- if [ -e /run/systemd/system ]; then
- # shellcheck source=./install-systemd-multi-user.sh
- . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh"
- else
- failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)"
- fi
+ # shellcheck source=./install-systemd-multi-user.sh
+ . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" # most of this works on non-systemd distros also
else
failure "Sorry, I don't know what to do on $(uname)"
fi
@@ -702,7 +724,10 @@ main() {
setup_default_profile
place_nix_configuration
- poly_configure_nix_daemon_service
+
+ if [ -e /run/systemd/system ]; then
+ poly_configure_nix_daemon_service
+ fi
trap finish_success EXIT
}
diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh
index 5824c2217..6efd8af18 100644
--- a/scripts/install-nix-from-closure.sh
+++ b/scripts/install-nix-from-closure.sh
@@ -35,7 +35,7 @@ fi
# Determine if we could use the multi-user installer or not
if [ "$(uname -s)" = "Darwin" ]; then
echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
-elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then
+elif [ "$(uname -s)" = "Linux" ]; then
echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2
fi
@@ -122,7 +122,7 @@ if [ "$(uname -s)" = "Darwin" ]; then
fi
if [ "$INSTALL_MODE" = "daemon" ]; then
- printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n'
+ printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n'
exec "$self/install-multi-user"
exit 0
fi
@@ -207,7 +207,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
if [ -w "$fn" ]; then
if ! grep -q "$p" "$fn"; then
echo "modifying $fn..." >&2
- echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+ echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
fi
added=1
break
@@ -218,7 +218,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
if [ -w "$fn" ]; then
if ! grep -q "$p" "$fn"; then
echo "modifying $fn..." >&2
- echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
+ echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
fi
added=1
break
diff --git a/shell.nix b/shell.nix
index 17aaa05ed..330df0ab6 100644
--- a/shell.nix
+++ b/shell.nix
@@ -1,25 +1,3 @@
-{ useClang ? false }:
-
-with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
-
-with import ./release-common.nix { inherit pkgs; };
-
-(if useClang then clangStdenv else stdenv).mkDerivation {
- name = "nix";
-
- buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
-
- inherit configureFlags;
-
- enableParallelBuilding = true;
-
- installFlags = "sysconfdir=$(out)/etc";
-
- shellHook =
- ''
- export prefix=$(pwd)/inst
- configureFlags+=" --prefix=$prefix"
- PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
- PATH=$prefix/bin:$PATH
- '';
-}
+(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
+ src = ./.;
+}).shellNix
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index e07117496..3579d8fff 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -33,7 +33,7 @@ std::string escapeUri(std::string uri)
static string currentLoad;
-static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
+static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
{
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
}
@@ -119,7 +119,7 @@ static int _main(int argc, char * * argv)
bool rightType = false;
Machine * bestMachine = nullptr;
- unsigned long long bestLoad = 0;
+ uint64_t bestLoad = 0;
for (auto & m : machines) {
debug("considering building on remote machine '%s'", m.storeUri);
@@ -130,8 +130,8 @@ static int _main(int argc, char * * argv)
m.mandatoryMet(requiredFeatures)) {
rightType = true;
AutoCloseFD free;
- unsigned long long load = 0;
- for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
+ uint64_t load = 0;
+ for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
auto slotLock = openSlotLock(m, slot);
if (lockFile(slotLock.get(), ltWrite, false)) {
if (!free) {
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 2e2a17b14..83854df49 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
Symbol file = state.symbols.create(filename);
- return { file, lineno, 0 };
+ return { foFile, file, lineno, 0 };
}
diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index c601d09c2..7eaa16c59 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -78,7 +78,7 @@ public:
if (!a)
throw Error({
.hint = hintfmt("attribute '%s' missing", name),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
return *a;
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 44baadd53..6b48ead1f 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -4,6 +4,8 @@
#include "util.hh"
#include "eval.hh"
#include "fetchers.hh"
+#include "registry.hh"
+#include "flake/flakeref.hh"
#include "store-api.hh"
namespace nix {
@@ -31,6 +33,27 @@ MixEvalArgs::MixEvalArgs()
.labels = {"path"},
.handler = {[&](std::string s) { searchPath.push_back(s); }}
});
+
+ addFlag({
+ .longName = "impure",
+ .description = "allow access to mutable paths and repositories",
+ .handler = {[&]() {
+ evalSettings.pureEval = false;
+ }},
+ });
+
+ addFlag({
+ .longName = "override-flake",
+ .description = "override a flake registry value",
+ .labels = {"original-ref", "resolved-ref"},
+ .handler = {[&](std::string _from, std::string _to) {
+ auto from = parseFlakeRef(_from, absPath("."));
+ auto to = parseFlakeRef(_to, absPath("."));
+ fetchers::Attrs extraAttrs;
+ if (to.subdir != "") extraAttrs["dir"] = to.subdir;
+ fetchers::overrideRegistry(from.input, to.input, extraAttrs);
+ }}
+ });
}
Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
@@ -53,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
if (isUri(s)) {
return state.store->toRealPath(
fetchers::downloadTarball(
- state.store, resolveUri(s), "source", false).storePath);
+ state.store, resolveUri(s), "source", false).first.storePath);
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
Path p = s.substr(1, s.size() - 2);
return state.findFile(p);
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
new file mode 100644
index 000000000..deb32484f
--- /dev/null
+++ b/src/libexpr/eval-cache.cc
@@ -0,0 +1,616 @@
+#include "eval-cache.hh"
+#include "sqlite.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+
+namespace nix::eval_cache {
+
+static const char * schema = R"sql(
+create table if not exists Attributes (
+ parent integer not null,
+ name text,
+ type integer not null,
+ value text,
+ context text,
+ primary key (parent, name)
+);
+)sql";
+
+struct AttrDb
+{
+ std::atomic_bool failed{false};
+
+ struct State
+ {
+ SQLite db;
+ SQLiteStmt insertAttribute;
+ SQLiteStmt insertAttributeWithContext;
+ SQLiteStmt queryAttribute;
+ SQLiteStmt queryAttributes;
+ std::unique_ptr<SQLiteTxn> txn;
+ };
+
+ std::unique_ptr<Sync<State>> _state;
+
+ AttrDb(const Hash & fingerprint)
+ : _state(std::make_unique<Sync<State>>())
+ {
+ auto state(_state->lock());
+
+ Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
+ createDirs(cacheDir);
+
+ Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
+
+ state->db = SQLite(dbPath);
+ state->db.isCache();
+ state->db.exec(schema);
+
+ state->insertAttribute.create(state->db,
+ "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
+
+ state->insertAttributeWithContext.create(state->db,
+ "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
+
+ state->queryAttribute.create(state->db,
+ "select rowid, type, value, context from Attributes where parent = ? and name = ?");
+
+ state->queryAttributes.create(state->db,
+ "select name from Attributes where parent = ?");
+
+ state->txn = std::make_unique<SQLiteTxn>(state->db);
+ }
+
+ ~AttrDb()
+ {
+ try {
+ auto state(_state->lock());
+ if (!failed)
+ state->txn->commit();
+ state->txn.reset();
+ } catch (...) {
+ ignoreException();
+ }
+ }
+
+ template<typename F>
+ AttrId doSQLite(F && fun)
+ {
+ if (failed) return 0;
+ try {
+ return fun();
+ } catch (SQLiteError &) {
+ ignoreException();
+ failed = true;
+ return 0;
+ }
+ }
+
+ AttrId setAttrs(
+ AttrKey key,
+ const std::vector<Symbol> & attrs)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::FullAttrs)
+ (0, false).exec();
+
+ AttrId rowId = state->db.getLastInsertedRowId();
+ assert(rowId);
+
+ for (auto & attr : attrs)
+ state->insertAttribute.use()
+ (rowId)
+ (attr)
+ (AttrType::Placeholder)
+ (0, false).exec();
+
+ return rowId;
+ });
+ }
+
+ AttrId setString(
+ AttrKey key,
+ std::string_view s,
+ const char * * context = nullptr)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ if (context) {
+ std::string ctx;
+ for (const char * * p = context; *p; ++p) {
+ if (p != context) ctx.push_back(' ');
+ ctx.append(*p);
+ }
+ state->insertAttributeWithContext.use()
+ (key.first)
+ (key.second)
+ (AttrType::String)
+ (s)
+ (ctx).exec();
+ } else {
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::String)
+ (s).exec();
+ }
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
+ AttrId setBool(
+ AttrKey key,
+ bool b)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::Bool)
+ (b ? 1 : 0).exec();
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
+ AttrId setPlaceholder(AttrKey key)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::Placeholder)
+ (0, false).exec();
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
+ AttrId setMissing(AttrKey key)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::Missing)
+ (0, false).exec();
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
+ AttrId setMisc(AttrKey key)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::Misc)
+ (0, false).exec();
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
+ AttrId setFailed(AttrKey key)
+ {
+ return doSQLite([&]()
+ {
+ auto state(_state->lock());
+
+ state->insertAttribute.use()
+ (key.first)
+ (key.second)
+ (AttrType::Failed)
+ (0, false).exec();
+
+ return state->db.getLastInsertedRowId();
+ });
+ }
+
+ std::optional<std::pair<AttrId, AttrValue>> getAttr(
+ AttrKey key,
+ SymbolTable & symbols)
+ {
+ auto state(_state->lock());
+
+ auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
+ if (!queryAttribute.next()) return {};
+
+ auto rowId = (AttrType) queryAttribute.getInt(0);
+ auto type = (AttrType) queryAttribute.getInt(1);
+
+ switch (type) {
+ case AttrType::Placeholder:
+ return {{rowId, placeholder_t()}};
+ case AttrType::FullAttrs: {
+ // FIXME: expensive, should separate this out.
+ std::vector<Symbol> attrs;
+ auto queryAttributes(state->queryAttributes.use()(rowId));
+ while (queryAttributes.next())
+ attrs.push_back(symbols.create(queryAttributes.getStr(0)));
+ return {{rowId, attrs}};
+ }
+ case AttrType::String: {
+ std::vector<std::pair<Path, std::string>> context;
+ if (!queryAttribute.isNull(3))
+ for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
+ context.push_back(decodeContext(s));
+ return {{rowId, string_t{queryAttribute.getStr(2), context}}};
+ }
+ case AttrType::Bool:
+ return {{rowId, queryAttribute.getInt(2) != 0}};
+ case AttrType::Missing:
+ return {{rowId, missing_t()}};
+ case AttrType::Misc:
+ return {{rowId, misc_t()}};
+ case AttrType::Failed:
+ return {{rowId, failed_t()}};
+ default:
+ throw Error("unexpected type in evaluation cache");
+ }
+ }
+};
+
+static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+{
+ try {
+ return std::make_shared<AttrDb>(fingerprint);
+ } catch (SQLiteError &) {
+ ignoreException();
+ return nullptr;
+ }
+}
+
+EvalCache::EvalCache(
+ std::optional<std::reference_wrapper<const Hash>> useCache,
+ EvalState & state,
+ RootLoader rootLoader)
+ : db(useCache ? makeAttrDb(*useCache) : nullptr)
+ , state(state)
+ , rootLoader(rootLoader)
+{
+}
+
+Value * EvalCache::getRootValue()
+{
+ if (!value) {
+ debug("getting root value");
+ value = allocRootValue(rootLoader());
+ }
+ return *value;
+}
+
+std::shared_ptr<AttrCursor> EvalCache::getRoot()
+{
+ return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
+}
+
+AttrCursor::AttrCursor(
+ ref<EvalCache> root,
+ Parent parent,
+ Value * value,
+ std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
+ : root(root), parent(parent), cachedValue(std::move(cachedValue))
+{
+ if (value)
+ _value = allocRootValue(value);
+}
+
+AttrKey AttrCursor::getKey()
+{
+ if (!parent)
+ return {0, root->state.sEpsilon};
+ if (!parent->first->cachedValue) {
+ parent->first->cachedValue = root->db->getAttr(
+ parent->first->getKey(), root->state.symbols);
+ assert(parent->first->cachedValue);
+ }
+ return {parent->first->cachedValue->first, parent->second};
+}
+
+Value & AttrCursor::getValue()
+{
+ if (!_value) {
+ if (parent) {
+ auto & vParent = parent->first->getValue();
+ root->state.forceAttrs(vParent);
+ auto attr = vParent.attrs->get(parent->second);
+ if (!attr)
+ throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
+ _value = allocRootValue(attr->value);
+ } else
+ _value = allocRootValue(root->getRootValue());
+ }
+ return **_value;
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath() const
+{
+ if (parent) {
+ auto attrPath = parent->first->getAttrPath();
+ attrPath.push_back(parent->second);
+ return attrPath;
+ } else
+ return {};
+}
+
+std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
+{
+ auto attrPath = getAttrPath();
+ attrPath.push_back(name);
+ return attrPath;
+}
+
+std::string AttrCursor::getAttrPathStr() const
+{
+ return concatStringsSep(".", getAttrPath());
+}
+
+std::string AttrCursor::getAttrPathStr(Symbol name) const
+{
+ return concatStringsSep(".", getAttrPath(name));
+}
+
+Value & AttrCursor::forceValue()
+{
+ debug("evaluating uncached attribute %s", getAttrPathStr());
+
+ auto & v = getValue();
+
+ try {
+ root->state.forceValue(v);
+ } catch (EvalError &) {
+ debug("setting '%s' to failed", getAttrPathStr());
+ if (root->db)
+ cachedValue = {root->db->setFailed(getKey()), failed_t()};
+ throw;
+ }
+
+ if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
+ if (v.type == tString)
+ cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
+ else if (v.type == tPath)
+ cachedValue = {root->db->setString(getKey(), v.path), v.path};
+ else if (v.type == tBool)
+ cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+ else if (v.type == tAttrs)
+ ; // FIXME: do something?
+ else
+ cachedValue = {root->db->setMisc(getKey()), misc_t()};
+ }
+
+ return v;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
+{
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+
+ if (cachedValue) {
+ if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+ for (auto & attr : *attrs)
+ if (attr == name)
+ return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
+ return nullptr;
+ } else if (std::get_if<placeholder_t>(&cachedValue->second)) {
+ auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
+ if (attr) {
+ if (std::get_if<missing_t>(&attr->second))
+ return nullptr;
+ else if (std::get_if<failed_t>(&attr->second))
+ throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
+ else
+ return std::make_shared<AttrCursor>(root,
+ std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
+ }
+ // Incomplete attrset, so need to fall thru and
+ // evaluate to see whether 'name' exists
+ } else
+ return nullptr;
+ //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+ }
+ }
+
+ auto & v = forceValue();
+
+ if (v.type != tAttrs)
+ return nullptr;
+ //throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+ auto attr = v.attrs->get(name);
+
+ if (!attr) {
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+ root->db->setMissing({cachedValue->first, name});
+ }
+ return nullptr;
+ }
+
+ std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
+ cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
+ }
+
+ return std::make_shared<AttrCursor>(
+ root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
+{
+ return maybeGetAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
+{
+ auto p = maybeGetAttr(name);
+ if (!p)
+ throw Error("attribute '%s' does not exist", getAttrPathStr(name));
+ return p;
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
+{
+ return getAttr(root->state.symbols.create(name));
+}
+
+std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
+{
+ auto res = shared_from_this();
+ for (auto & attr : attrPath) {
+ res = res->maybeGetAttr(attr);
+ if (!res) return {};
+ }
+ return res;
+}
+
+std::string AttrCursor::getString()
+{
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+ if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+ if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+ debug("using cached string attribute '%s'", getAttrPathStr());
+ return s->first;
+ } else
+ throw TypeError("'%s' is not a string", getAttrPathStr());
+ }
+ }
+
+ auto & v = forceValue();
+
+ if (v.type != tString && v.type != tPath)
+ throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+
+ return v.type == tString ? v.string.s : v.path;
+}
+
+string_t AttrCursor::getStringWithContext()
+{
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+ if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+ if (auto s = std::get_if<string_t>(&cachedValue->second)) {
+ debug("using cached string attribute '%s'", getAttrPathStr());
+ return *s;
+ } else
+ throw TypeError("'%s' is not a string", getAttrPathStr());
+ }
+ }
+
+ auto & v = forceValue();
+
+ if (v.type == tString)
+ return {v.string.s, v.getContext()};
+ else if (v.type == tPath)
+ return {v.path, {}};
+ else
+ throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+}
+
+bool AttrCursor::getBool()
+{
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+ if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+ if (auto b = std::get_if<bool>(&cachedValue->second)) {
+ debug("using cached Boolean attribute '%s'", getAttrPathStr());
+ return *b;
+ } else
+ throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+ }
+ }
+
+ auto & v = forceValue();
+
+ if (v.type != tBool)
+ throw TypeError("'%s' is not a Boolean", getAttrPathStr());
+
+ return v.boolean;
+}
+
+std::vector<Symbol> AttrCursor::getAttrs()
+{
+ if (root->db) {
+ if (!cachedValue)
+ cachedValue = root->db->getAttr(getKey(), root->state.symbols);
+ if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+ if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
+ debug("using cached attrset attribute '%s'", getAttrPathStr());
+ return *attrs;
+ } else
+ throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+ }
+ }
+
+ auto & v = forceValue();
+
+ if (v.type != tAttrs)
+ throw TypeError("'%s' is not an attribute set", getAttrPathStr());
+
+ std::vector<Symbol> attrs;
+ for (auto & attr : *getValue().attrs)
+ attrs.push_back(attr.name);
+ std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
+ return (const string &) a < (const string &) b;
+ });
+
+ if (root->db)
+ cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
+
+ return attrs;
+}
+
+bool AttrCursor::isDerivation()
+{
+ auto aType = maybeGetAttr("type");
+ return aType && aType->getString() == "derivation";
+}
+
+StorePath AttrCursor::forceDerivation()
+{
+ auto aDrvPath = getAttr(root->state.sDrvPath);
+ auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
+ if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
+ /* The eval cache contains 'drvPath', but the actual path has
+ been garbage-collected. So force it to be regenerated. */
+ aDrvPath->forceValue();
+ if (!root->state.store->isValidPath(drvPath))
+ throw Error("don't know how to recreate store derivation '%s'!",
+ root->state.store->printStorePath(drvPath));
+ }
+ return drvPath;
+}
+
+}
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
new file mode 100644
index 000000000..afee85fa9
--- /dev/null
+++ b/src/libexpr/eval-cache.hh
@@ -0,0 +1,121 @@
+#pragma once
+
+#include "sync.hh"
+#include "hash.hh"
+#include "eval.hh"
+
+#include <functional>
+#include <variant>
+
+namespace nix::eval_cache {
+
+class AttrDb;
+class AttrCursor;
+
+class EvalCache : public std::enable_shared_from_this<EvalCache>
+{
+ friend class AttrCursor;
+
+ std::shared_ptr<AttrDb> db;
+ EvalState & state;
+ typedef std::function<Value *()> RootLoader;
+ RootLoader rootLoader;
+ RootValue value;
+
+ Value * getRootValue();
+
+public:
+
+ EvalCache(
+ std::optional<std::reference_wrapper<const Hash>> useCache,
+ EvalState & state,
+ RootLoader rootLoader);
+
+ std::shared_ptr<AttrCursor> getRoot();
+};
+
+enum AttrType {
+ Placeholder = 0,
+ FullAttrs = 1,
+ String = 2,
+ Missing = 3,
+ Misc = 4,
+ Failed = 5,
+ Bool = 6,
+};
+
+struct placeholder_t {};
+struct missing_t {};
+struct misc_t {};
+struct failed_t {};
+typedef uint64_t AttrId;
+typedef std::pair<AttrId, Symbol> AttrKey;
+typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+
+typedef std::variant<
+ std::vector<Symbol>,
+ string_t,
+ placeholder_t,
+ missing_t,
+ misc_t,
+ failed_t,
+ bool
+ > AttrValue;
+
+class AttrCursor : public std::enable_shared_from_this<AttrCursor>
+{
+ friend class EvalCache;
+
+ ref<EvalCache> root;
+ typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
+ Parent parent;
+ RootValue _value;
+ std::optional<std::pair<AttrId, AttrValue>> cachedValue;
+
+ AttrKey getKey();
+
+ Value & getValue();
+
+public:
+
+ AttrCursor(
+ ref<EvalCache> root,
+ Parent parent,
+ Value * value = nullptr,
+ std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
+
+ std::vector<Symbol> getAttrPath() const;
+
+ std::vector<Symbol> getAttrPath(Symbol name) const;
+
+ std::string getAttrPathStr() const;
+
+ std::string getAttrPathStr(Symbol name) const;
+
+ std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
+
+ std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
+
+ std::shared_ptr<AttrCursor> getAttr(Symbol name);
+
+ std::shared_ptr<AttrCursor> getAttr(std::string_view name);
+
+ std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
+
+ std::string getString();
+
+ string_t getStringWithContext();
+
+ bool getBool();
+
+ std::vector<Symbol> getAttrs();
+
+ bool isDerivation();
+
+ Value & forceValue();
+
+ /* Force creation of the .drv file in the Nix store. */
+ StorePath forceDerivation();
+};
+
+}
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 3d544c903..30f6ec7db 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -11,7 +11,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
{
throw EvalError({
.hint = hintfmt(s),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -25,7 +25,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
{
throw TypeError({
.hint = hintfmt(s, showType(v)),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index b90a64357..7a2f55504 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -199,6 +199,18 @@ string showType(const Value & v)
}
+bool Value::isTrivial() const
+{
+ return
+ type != tApp
+ && type != tPrimOpApp
+ && (type != tThunk
+ || (dynamic_cast<ExprAttrs *>(thunk.expr)
+ && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
+ || dynamic_cast<ExprLambda *>(thunk.expr));
+}
+
+
#if HAVE_BOEHMGC
/* Called when the Boehm GC runs out of memory. */
static void * oomHandler(size_t requested)
@@ -337,6 +349,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
, sRecurseForDerivations(symbols.create("recurseForDerivations"))
+ , sDescription(symbols.create("description"))
+ , sSelf(symbols.create("self"))
+ , sEpsilon(symbols.create(""))
, repair(NoRepair)
, store(store)
, baseEnv(allocEnv(128))
@@ -366,7 +381,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
if (store->isInStore(r.second)) {
StorePathSet closure;
- store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
+ store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
allowedPaths->insert(store->printStorePath(path));
} else
@@ -529,7 +544,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
{
throw EvalError({
.hint = hintfmt(s, s2),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -542,7 +557,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
{
throw EvalError({
.hint = hintfmt(s, s2, s3),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -551,7 +566,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
// p1 is where the error occurred; p2 is a position mentioned in the message.
throw EvalError({
.hint = hintfmt(s, sym, p2),
- .nixCode = NixCode { .errPos = p1 }
+ .errPos = p1
});
}
@@ -559,7 +574,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
{
throw TypeError({
.hint = hintfmt(s),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -572,7 +587,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
{
throw TypeError({
.hint = hintfmt(s, fun.showNamePos(), s2),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -580,7 +595,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
{
throw AssertionError({
.hint = hintfmt(s, s1),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -588,23 +603,18 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
{
throw UndefinedVarError({
.hint = hintfmt(s, s1),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
+LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
{
- e.addPrefix(format(s) % s2);
+ e.addTrace(std::nullopt, s, s2);
}
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
+LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
{
- e.addPrefix(format(s) % fun.showNamePos() % pos);
-}
-
-LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
-{
- e.addPrefix(format(s) % s2 % pos);
+ e.addTrace(pos, s, s2);
}
@@ -787,7 +797,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
}
-void EvalState::evalFile(const Path & path_, Value & v)
+void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
{
auto path = checkSourcePath(path_);
@@ -816,9 +826,14 @@ void EvalState::evalFile(const Path & path_, Value & v)
fileParseCache[path2] = e;
try {
+ // Enforce that 'flake.nix' is a direct attrset, not a
+ // computation.
+ if (mustBeTrivial &&
+ !(dynamic_cast<ExprAttrs *>(e)))
+ throw Error("file '%s' must be an attribute set", path);
eval(e, v);
} catch (Error & e) {
- addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
+ addErrorTrace(e, "while evaluating the file '%1%':", path2);
throw;
}
@@ -1068,8 +1083,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
} catch (Error & e) {
if (pos2 && pos2->file != state.sDerivationNix)
- addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
- showAttrPath(state, env, attrPath), *pos2);
+ addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
+ showAttrPath(state, env, attrPath));
throw;
}
@@ -1237,11 +1252,15 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
/* Evaluate the body. This is conditional on showTrace, because
catching exceptions makes this function not tail-recursive. */
- if (settings.showTrace)
+ if (loggerSettings.showTrace.get())
try {
lambda.body->eval(*this, env2, v);
} catch (Error & e) {
- addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
+ addErrorTrace(e, lambda.pos, "while evaluating %s",
+ (lambda.name.set()
+ ? "'" + (string) lambda.name + "'"
+ : "anonymous lambdaction"));
+ addErrorTrace(e, pos, "from call site%s", "");
throw;
}
else
@@ -1516,7 +1535,7 @@ void EvalState::forceValueDeep(Value & v)
try {
recurse(*i.value);
} catch (Error & e) {
- addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
+ addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
throw;
}
}
@@ -1587,6 +1606,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
}
+/* Decode a context string ‘!<name>!<path>’ into a pair <path,
+ name>. */
+std::pair<string, string> decodeContext(std::string_view s)
+{
+ if (s.at(0) == '!') {
+ size_t index = s.find("!", 1);
+ return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+ } else
+ return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+}
+
+
void copyContext(const Value & v, PathSet & context)
{
if (v.string.context)
@@ -1595,6 +1626,17 @@ void copyContext(const Value & v, PathSet & context)
}
+std::vector<std::pair<Path, std::string>> Value::getContext()
+{
+ std::vector<std::pair<Path, std::string>> res;
+ assert(type == tString);
+ if (string.context)
+ for (const char * * p = string.context; *p; ++p)
+ res.push_back(decodeContext(*p));
+ return res;
+}
+
+
string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
{
string s = forceString(v, pos);
@@ -1936,7 +1978,7 @@ string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, boo
{
throw TypeError({
.hint = hintfmt("cannot coerce %1% to a string", showType()),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 863365259..8986952e3 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -4,13 +4,13 @@
#include "value.hh"
#include "nixexpr.hh"
#include "symbol-table.hh"
-#include "hash.hh"
#include "config.hh"
#include <regex>
#include <map>
#include <optional>
#include <unordered_map>
+#include <mutex>
namespace nix {
@@ -75,7 +75,8 @@ public:
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
- sRecurseForDerivations;
+ sRecurseForDerivations,
+ sDescription, sSelf, sEpsilon;
Symbol sDerivationNix;
/* If set, force copying files to the Nix store even if they
@@ -90,6 +91,7 @@ public:
const ref<Store> store;
+
private:
SrcToStore srcToStore;
@@ -152,8 +154,9 @@ public:
Expr * parseStdin();
/* Evaluate an expression read from the given file to normal
- form. */
- void evalFile(const Path & path, Value & v);
+ form. Optionally enforce that the top-level expression is
+ trivial (i.e. doesn't require arbitrary computation). */
+ void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
void resetFileCache();
@@ -250,7 +253,7 @@ private:
friend struct ExprAttrs;
friend struct ExprLet;
- Expr * parse(const char * text, const Path & path,
+ Expr * parse(const char * text, FileOrigin origin, const Path & path,
const Path & basePath, StaticEnv & staticEnv);
public:
@@ -330,7 +333,7 @@ string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
-std::pair<string, string> decodeContext(const string & s);
+std::pair<string, string> decodeContext(std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);
diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix
new file mode 100644
index 000000000..932ac5e90
--- /dev/null
+++ b/src/libexpr/flake/call-flake.nix
@@ -0,0 +1,56 @@
+lockFileStr: rootSrc: rootSubdir:
+
+let
+
+ lockFile = builtins.fromJSON lockFileStr;
+
+ allNodes =
+ builtins.mapAttrs
+ (key: node:
+ let
+
+ sourceInfo =
+ if key == lockFile.root
+ then rootSrc
+ else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
+
+ subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
+
+ flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
+
+ inputs = builtins.mapAttrs
+ (inputName: inputSpec: allNodes.${resolveInput inputSpec})
+ (node.inputs or {});
+
+ # Resolve a input spec into a node name. An input spec is
+ # either a node name, or a 'follows' path from the root
+ # node.
+ resolveInput = inputSpec:
+ if builtins.isList inputSpec
+ then getInputByPath lockFile.root inputSpec
+ else inputSpec;
+
+ # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
+ # root node, returning the final node.
+ getInputByPath = nodeName: path:
+ if path == []
+ then nodeName
+ else
+ getInputByPath
+ # Since this could be a 'follows' input, call resolveInput.
+ (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
+ (builtins.tail path);
+
+ outputs = flake.outputs (inputs // { self = result; });
+
+ result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
+ in
+ if node.flake or true then
+ assert builtins.isFunction flake.outputs;
+ result
+ else
+ sourceInfo
+ )
+ lockFile.nodes;
+
+in allNodes.${lockFile.root}
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
new file mode 100644
index 000000000..01f464859
--- /dev/null
+++ b/src/libexpr/flake/flake.cc
@@ -0,0 +1,609 @@
+#include "flake.hh"
+#include "lockfile.hh"
+#include "primops.hh"
+#include "eval-inline.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "finally.hh"
+
+namespace nix {
+
+using namespace flake;
+
+namespace flake {
+
+typedef std::pair<Tree, FlakeRef> FetchedFlake;
+typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
+
+static std::optional<FetchedFlake> lookupInFlakeCache(
+ const FlakeCache & flakeCache,
+ const FlakeRef & flakeRef)
+{
+ // FIXME: inefficient.
+ for (auto & i : flakeCache) {
+ if (flakeRef == i.first) {
+ debug("mapping '%s' to previously seen input '%s' -> '%s",
+ flakeRef, i.first, i.second.second);
+ return i.second;
+ }
+ }
+
+ return std::nullopt;
+}
+
+static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
+ EvalState & state,
+ const FlakeRef & originalRef,
+ bool allowLookup,
+ FlakeCache & flakeCache)
+{
+ auto fetched = lookupInFlakeCache(flakeCache, originalRef);
+ FlakeRef resolvedRef = originalRef;
+
+ if (!fetched) {
+ if (originalRef.input.isDirect()) {
+ fetched.emplace(originalRef.fetchTree(state.store));
+ } else {
+ if (allowLookup) {
+ resolvedRef = originalRef.resolve(state.store);
+ auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
+ if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
+ flakeCache.push_back({resolvedRef, fetchedResolved.value()});
+ fetched.emplace(fetchedResolved.value());
+ }
+ else {
+ throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
+ }
+ }
+ flakeCache.push_back({originalRef, fetched.value()});
+ }
+
+ auto [tree, lockedRef] = fetched.value();
+
+ debug("got tree '%s' from '%s'",
+ state.store->printStorePath(tree.storePath), lockedRef);
+
+ if (state.allowedPaths)
+ state.allowedPaths->insert(tree.actualPath);
+
+ assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
+
+ return {std::move(tree), resolvedRef, lockedRef};
+}
+
+static void expectType(EvalState & state, ValueType type,
+ Value & value, const Pos & pos)
+{
+ if (value.type == tThunk && value.isTrivial())
+ state.forceValue(value, pos);
+ if (value.type != type)
+ throw Error("expected %s but got %s at %s",
+ showType(type), showType(value.type), pos);
+}
+
+static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+ EvalState & state, Value * value, const Pos & pos);
+
+static FlakeInput parseFlakeInput(EvalState & state,
+ const std::string & inputName, Value * value, const Pos & pos)
+{
+ expectType(state, tAttrs, *value, pos);
+
+ FlakeInput input;
+
+ auto sInputs = state.symbols.create("inputs");
+ auto sUrl = state.symbols.create("url");
+ auto sFlake = state.symbols.create("flake");
+ auto sFollows = state.symbols.create("follows");
+
+ fetchers::Attrs attrs;
+ std::optional<std::string> url;
+
+ for (nix::Attr attr : *(value->attrs)) {
+ try {
+ if (attr.name == sUrl) {
+ expectType(state, tString, *attr.value, *attr.pos);
+ url = attr.value->string.s;
+ attrs.emplace("url", *url);
+ } else if (attr.name == sFlake) {
+ expectType(state, tBool, *attr.value, *attr.pos);
+ input.isFlake = attr.value->boolean;
+ } else if (attr.name == sInputs) {
+ input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
+ } else if (attr.name == sFollows) {
+ expectType(state, tString, *attr.value, *attr.pos);
+ input.follows = parseInputPath(attr.value->string.s);
+ } else {
+ state.forceValue(*attr.value);
+ if (attr.value->type == tString)
+ attrs.emplace(attr.name, attr.value->string.s);
+ else
+ throw TypeError("flake input attribute '%s' is %s while a string is expected",
+ attr.name, showType(*attr.value));
+ }
+ } catch (Error & e) {
+ e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
+ throw;
+ }
+ }
+
+ if (attrs.count("type"))
+ try {
+ input.ref = FlakeRef::fromAttrs(attrs);
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("in flake input"));
+ throw;
+ }
+ else {
+ attrs.erase("url");
+ if (!attrs.empty())
+ throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
+ if (url)
+ input.ref = parseFlakeRef(*url, {}, true);
+ }
+
+ if (!input.follows && !input.ref)
+ input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}});
+
+ return input;
+}
+
+static std::map<FlakeId, FlakeInput> parseFlakeInputs(
+ EvalState & state, Value * value, const Pos & pos)
+{
+ std::map<FlakeId, FlakeInput> inputs;
+
+ expectType(state, tAttrs, *value, pos);
+
+ for (nix::Attr & inputAttr : *(*value).attrs) {
+ inputs.emplace(inputAttr.name,
+ parseFlakeInput(state,
+ inputAttr.name,
+ inputAttr.value,
+ *inputAttr.pos));
+ }
+
+ return inputs;
+}
+
+static Flake getFlake(
+ EvalState & state,
+ const FlakeRef & originalRef,
+ bool allowLookup,
+ FlakeCache & flakeCache)
+{
+ auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+ state, originalRef, allowLookup, flakeCache);
+
+ // Guard against symlink attacks.
+ auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
+ if (!isInDir(flakeFile, sourceInfo.actualPath))
+ throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
+ lockedRef, state.store->printStorePath(sourceInfo.storePath));
+
+ Flake flake {
+ .originalRef = originalRef,
+ .resolvedRef = resolvedRef,
+ .lockedRef = lockedRef,
+ .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
+ };
+
+ if (!pathExists(flakeFile))
+ throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
+
+ Value vInfo;
+ state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
+
+ expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
+
+ auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
+
+ if (vInfo.attrs->get(sEdition))
+ warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
+
+ if (auto description = vInfo.attrs->get(state.sDescription)) {
+ expectType(state, tString, *description->value, *description->pos);
+ flake.description = description->value->string.s;
+ }
+
+ auto sInputs = state.symbols.create("inputs");
+
+ if (auto inputs = vInfo.attrs->get(sInputs))
+ flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
+
+ auto sOutputs = state.symbols.create("outputs");
+
+ if (auto outputs = vInfo.attrs->get(sOutputs)) {
+ expectType(state, tLambda, *outputs->value, *outputs->pos);
+ flake.vOutputs = allocRootValue(outputs->value);
+
+ if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
+ for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
+ if (formal.name != state.sSelf)
+ flake.inputs.emplace(formal.name, FlakeInput {
+ .ref = parseFlakeRef(formal.name)
+ });
+ }
+ }
+
+ } else
+ throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
+
+ for (auto & attr : *vInfo.attrs) {
+ if (attr.name != sEdition &&
+ attr.name != state.sDescription &&
+ attr.name != sInputs &&
+ attr.name != sOutputs)
+ throw Error("flake '%s' has an unsupported attribute '%s', at %s",
+ lockedRef, attr.name, *attr.pos);
+ }
+
+ return flake;
+}
+
+Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
+{
+ FlakeCache flakeCache;
+ return getFlake(state, originalRef, allowLookup, flakeCache);
+}
+
+/* Compute an in-memory lock file for the specified top-level flake,
+ and optionally write it to file, it the flake is writable. */
+LockedFlake lockFlake(
+ EvalState & state,
+ const FlakeRef & topRef,
+ const LockFlags & lockFlags)
+{
+ settings.requireExperimentalFeature("flakes");
+
+ FlakeCache flakeCache;
+
+ auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
+
+ // FIXME: symlink attack
+ auto oldLockFile = LockFile::read(
+ flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
+
+ debug("old lock file: %s", oldLockFile);
+
+ // FIXME: check whether all overrides are used.
+ std::map<InputPath, FlakeInput> overrides;
+ std::set<InputPath> overridesUsed, updatesUsed;
+
+ for (auto & i : lockFlags.inputOverrides)
+ overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+
+ LockFile newLockFile;
+
+ std::vector<FlakeRef> parents;
+
+ std::function<void(
+ const FlakeInputs & flakeInputs,
+ std::shared_ptr<Node> node,
+ const InputPath & inputPathPrefix,
+ std::shared_ptr<const Node> oldNode)>
+ computeLocks;
+
+ computeLocks = [&](
+ const FlakeInputs & flakeInputs,
+ std::shared_ptr<Node> node,
+ const InputPath & inputPathPrefix,
+ std::shared_ptr<const Node> oldNode)
+ {
+ debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
+
+ /* Get the overrides (i.e. attributes of the form
+ 'inputs.nixops.inputs.nixpkgs.url = ...'). */
+ // FIXME: check this
+ for (auto & [id, input] : flake.inputs) {
+ for (auto & [idOverride, inputOverride] : input.overrides) {
+ auto inputPath(inputPathPrefix);
+ inputPath.push_back(id);
+ inputPath.push_back(idOverride);
+ overrides.insert_or_assign(inputPath, inputOverride);
+ }
+ }
+
+ /* Go over the flake inputs, resolve/fetch them if
+ necessary (i.e. if they're new or the flakeref changed
+ from what's in the lock file). */
+ for (auto & [id, input2] : flakeInputs) {
+ auto inputPath(inputPathPrefix);
+ inputPath.push_back(id);
+ auto inputPathS = printInputPath(inputPath);
+ debug("computing input '%s'", inputPathS);
+
+ /* Do we have an override for this input from one of the
+ ancestors? */
+ auto i = overrides.find(inputPath);
+ bool hasOverride = i != overrides.end();
+ if (hasOverride) overridesUsed.insert(inputPath);
+ auto & input = hasOverride ? i->second : input2;
+
+ /* Resolve 'follows' later (since it may refer to an input
+ path we haven't processed yet. */
+ if (input.follows) {
+ InputPath target;
+ if (hasOverride || input.absolute)
+ /* 'follows' from an override is relative to the
+ root of the graph. */
+ target = *input.follows;
+ else {
+ /* Otherwise, it's relative to the current flake. */
+ target = inputPathPrefix;
+ for (auto & i : *input.follows) target.push_back(i);
+ }
+ debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
+ node->inputs.insert_or_assign(id, target);
+ continue;
+ }
+
+ assert(input.ref);
+
+ /* Do we have an entry in the existing lock file? And we
+ don't have a --update-input flag for this input? */
+ std::shared_ptr<LockedNode> oldLock;
+
+ updatesUsed.insert(inputPath);
+
+ if (oldNode && !lockFlags.inputUpdates.count(inputPath))
+ if (auto oldLock2 = get(oldNode->inputs, id))
+ if (auto oldLock3 = std::get_if<0>(&*oldLock2))
+ oldLock = *oldLock3;
+
+ if (oldLock
+ && oldLock->originalRef == *input.ref
+ && !hasOverride)
+ {
+ debug("keeping existing input '%s'", inputPathS);
+
+ /* Copy the input from the old lock since its flakeref
+ didn't change and there is no override from a
+ higher level flake. */
+ auto childNode = std::make_shared<LockedNode>(
+ oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
+
+ node->inputs.insert_or_assign(id, childNode);
+
+ /* If we have an --update-input flag for an input
+ of this input, then we must fetch the flake to
+ to update it. */
+ auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
+
+ auto hasChildUpdate =
+ lb != lockFlags.inputUpdates.end()
+ && lb->size() > inputPath.size()
+ && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
+
+ if (hasChildUpdate) {
+ auto inputFlake = getFlake(
+ state, oldLock->lockedRef, false, flakeCache);
+ computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+ } else {
+ /* No need to fetch this flake, we can be
+ lazy. However there may be new overrides on the
+ inputs of this flake, so we need to check
+ those. */
+ FlakeInputs fakeInputs;
+
+ for (auto & i : oldLock->inputs) {
+ if (auto lockedNode = std::get_if<0>(&i.second)) {
+ fakeInputs.emplace(i.first, FlakeInput {
+ .ref = (*lockedNode)->originalRef,
+ .isFlake = (*lockedNode)->isFlake,
+ });
+ } else if (auto follows = std::get_if<1>(&i.second)) {
+ fakeInputs.emplace(i.first, FlakeInput {
+ .follows = *follows,
+ .absolute = true
+ });
+ }
+ }
+
+ computeLocks(fakeInputs, childNode, inputPath, oldLock);
+ }
+
+ } else {
+ /* We need to create a new lock file entry. So fetch
+ this input. */
+ debug("creating new input '%s'", inputPathS);
+
+ if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
+ throw Error("cannot update flake input '%s' in pure mode", inputPathS);
+
+ if (input.isFlake) {
+ auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
+
+ /* Note: in case of an --override-input, we use
+ the *original* ref (input2.ref) for the
+ "original" field, rather than the
+ override. This ensures that the override isn't
+ nuked the next time we update the lock
+ file. That is, overrides are sticky unless you
+ use --no-write-lock-file. */
+ auto childNode = std::make_shared<LockedNode>(
+ inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
+
+ node->inputs.insert_or_assign(id, childNode);
+
+ /* Guard against circular flake imports. */
+ for (auto & parent : parents)
+ if (parent == *input.ref)
+ throw Error("found circular import of flake '%s'", parent);
+ parents.push_back(*input.ref);
+ Finally cleanup([&]() { parents.pop_back(); });
+
+ /* Recursively process the inputs of this
+ flake. Also, unless we already have this flake
+ in the top-level lock file, use this flake's
+ own lock file. */
+ computeLocks(
+ inputFlake.inputs, childNode, inputPath,
+ oldLock
+ ? std::dynamic_pointer_cast<const Node>(oldLock)
+ : LockFile::read(
+ inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+ }
+
+ else {
+ auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+ state, *input.ref, lockFlags.useRegistries, flakeCache);
+ node->inputs.insert_or_assign(id,
+ std::make_shared<LockedNode>(lockedRef, *input.ref, false));
+ }
+ }
+ }
+ };
+
+ computeLocks(
+ flake.inputs, newLockFile.root, {},
+ lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+
+ for (auto & i : lockFlags.inputOverrides)
+ if (!overridesUsed.count(i.first))
+ warn("the flag '--override-input %s %s' does not match any input",
+ printInputPath(i.first), i.second);
+
+ for (auto & i : lockFlags.inputUpdates)
+ if (!updatesUsed.count(i))
+ warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+
+ /* Check 'follows' inputs. */
+ newLockFile.check();
+
+ debug("new lock file: %s", newLockFile);
+
+ /* Check whether we need to / can write the new lock file. */
+ if (!(newLockFile == oldLockFile)) {
+
+ auto diff = LockFile::diff(oldLockFile, newLockFile);
+
+ if (lockFlags.writeLockFile) {
+ if (auto sourcePath = topRef.input.getSourcePath()) {
+ if (!newLockFile.isImmutable()) {
+ if (settings.warnDirty)
+ warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
+ } else {
+ if (!lockFlags.updateLockFile)
+ throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
+
+ auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+
+ auto path = *sourcePath + "/" + relPath;
+
+ bool lockFileExists = pathExists(path);
+
+ if (lockFileExists) {
+ auto s = chomp(diff);
+ if (s.empty())
+ warn("updating lock file '%s'", path);
+ else
+ warn("updating lock file '%s':\n%s", path, s);
+ } else
+ warn("creating lock file '%s'", path);
+
+ newLockFile.write(path);
+
+ topRef.input.markChangedFile(
+ (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
+ lockFlags.commitLockFile
+ ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
+ relPath, lockFileExists ? "Update" : "Add", diff))
+ : std::nullopt);
+
+ /* Rewriting the lockfile changed the top-level
+ repo, so we should re-read it. FIXME: we could
+ also just clear the 'rev' field... */
+ auto prevLockedRef = flake.lockedRef;
+ FlakeCache dummyCache;
+ flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
+
+ if (lockFlags.commitLockFile &&
+ flake.lockedRef.input.getRev() &&
+ prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
+ warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
+
+ /* Make sure that we picked up the change,
+ i.e. the tree should usually be dirty
+ now. Corner case: we could have reverted from a
+ dirty to a clean tree! */
+ if (flake.lockedRef.input == prevLockedRef.input
+ && !flake.lockedRef.input.isImmutable())
+ throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
+ }
+ } else
+ throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
+ } else
+ warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
+ }
+
+ return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+}
+
+void callFlake(EvalState & state,
+ const LockedFlake & lockedFlake,
+ Value & vRes)
+{
+ auto vLocks = state.allocValue();
+ auto vRootSrc = state.allocValue();
+ auto vRootSubdir = state.allocValue();
+ auto vTmp1 = state.allocValue();
+ auto vTmp2 = state.allocValue();
+
+ mkString(*vLocks, lockedFlake.lockFile.to_string());
+
+ emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
+
+ mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
+
+ static RootValue vCallFlake = nullptr;
+
+ if (!vCallFlake) {
+ vCallFlake = allocRootValue(state.allocValue());
+ state.eval(state.parseExprFromString(
+ #include "call-flake.nix.gen.hh"
+ , "/"), **vCallFlake);
+ }
+
+ state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
+ state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
+ state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
+}
+
+static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
+ auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
+ if (evalSettings.pureEval && !flakeRef.input.isImmutable())
+ throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
+
+ callFlake(state,
+ lockFlake(state, flakeRef,
+ LockFlags {
+ .updateLockFile = false,
+ .useRegistries = !evalSettings.pureEval,
+ .allowMutable = !evalSettings.pureEval,
+ }),
+ v);
+}
+
+static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
+
+}
+
+Fingerprint LockedFlake::getFingerprint() const
+{
+ // FIXME: as an optimization, if the flake contains a lock file
+ // and we haven't changed it, then it's sufficient to use
+ // flake.sourceInfo.storePath for the fingerprint.
+ return hashString(htSHA256,
+ fmt("%s;%d;%d;%s",
+ flake.sourceInfo->storePath.to_string(),
+ flake.lockedRef.input.getRevCount().value_or(0),
+ flake.lockedRef.input.getLastModified().value_or(0),
+ lockFile));
+}
+
+Flake::~Flake() { }
+
+}
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
new file mode 100644
index 000000000..c2bb2888b
--- /dev/null
+++ b/src/libexpr/flake/flake.hh
@@ -0,0 +1,111 @@
+#pragma once
+
+#include "types.hh"
+#include "flakeref.hh"
+#include "lockfile.hh"
+#include "value.hh"
+
+namespace nix {
+
+class EvalState;
+
+namespace fetchers { struct Tree; }
+
+namespace flake {
+
+struct FlakeInput;
+
+typedef std::map<FlakeId, FlakeInput> FlakeInputs;
+
+struct FlakeInput
+{
+ std::optional<FlakeRef> ref;
+ bool isFlake = true;
+ std::optional<InputPath> follows;
+ bool absolute = false; // whether 'follows' is relative to the flake root
+ FlakeInputs overrides;
+};
+
+struct Flake
+{
+ FlakeRef originalRef;
+ FlakeRef resolvedRef;
+ FlakeRef lockedRef;
+ std::optional<std::string> description;
+ std::shared_ptr<const fetchers::Tree> sourceInfo;
+ FlakeInputs inputs;
+ RootValue vOutputs;
+ ~Flake();
+};
+
+Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
+
+/* Fingerprint of a locked flake; used as a cache key. */
+typedef Hash Fingerprint;
+
+struct LockedFlake
+{
+ Flake flake;
+ LockFile lockFile;
+
+ Fingerprint getFingerprint() const;
+};
+
+struct LockFlags
+{
+ /* Whether to ignore the existing lock file, creating a new one
+ from scratch. */
+ bool recreateLockFile = false;
+
+ /* Whether to update the lock file at all. If set to false, if any
+ change to the lock file is needed (e.g. when an input has been
+ added to flake.nix), you get a fatal error. */
+ bool updateLockFile = true;
+
+ /* Whether to write the lock file to disk. If set to true, if the
+ any changes to the lock file are needed and the flake is not
+ writable (i.e. is not a local Git working tree or similar), you
+ get a fatal error. If set to false, Nix will use the modified
+ lock file in memory only, without writing it to disk. */
+ bool writeLockFile = true;
+
+ /* Whether to use the registries to lookup indirect flake
+ references like 'nixpkgs'. */
+ bool useRegistries = true;
+
+ /* Whether mutable flake references (i.e. those without a Git
+ revision or similar) without a corresponding lock are
+ allowed. Mutable flake references with a lock are always
+ allowed. */
+ bool allowMutable = true;
+
+ /* Whether to commit changes to flake.lock. */
+ bool commitLockFile = false;
+
+ /* Flake inputs to be overriden. */
+ std::map<InputPath, FlakeRef> inputOverrides;
+
+ /* Flake inputs to be updated. This means that any existing lock
+ for those inputs will be ignored. */
+ std::set<InputPath> inputUpdates;
+};
+
+LockedFlake lockFlake(
+ EvalState & state,
+ const FlakeRef & flakeRef,
+ const LockFlags & lockFlags);
+
+void callFlake(
+ EvalState & state,
+ const LockedFlake & lockedFlake,
+ Value & v);
+
+}
+
+void emitTreeAttrs(
+ EvalState & state,
+ const fetchers::Tree & tree,
+ const fetchers::Input & input,
+ Value & v, bool emptyRevFallback = false);
+
+}
diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
new file mode 100644
index 000000000..6363446f6
--- /dev/null
+++ b/src/libexpr/flake/flakeref.cc
@@ -0,0 +1,204 @@
+#include "flakeref.hh"
+#include "store-api.hh"
+#include "url.hh"
+#include "fetchers.hh"
+#include "registry.hh"
+
+namespace nix {
+
+#if 0
+// 'dir' path elements cannot start with a '.'. We also reject
+// potentially dangerous characters like ';'.
+const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
+const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
+#endif
+
+std::string FlakeRef::to_string() const
+{
+ auto url = input.toURL();
+ if (subdir != "")
+ url.query.insert_or_assign("dir", subdir);
+ return url.to_string();
+}
+
+fetchers::Attrs FlakeRef::toAttrs() const
+{
+ auto attrs = input.toAttrs();
+ if (subdir != "")
+ attrs.emplace("dir", subdir);
+ return attrs;
+}
+
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
+{
+ str << flakeRef.to_string();
+ return str;
+}
+
+bool FlakeRef::operator ==(const FlakeRef & other) const
+{
+ return input == other.input && subdir == other.subdir;
+}
+
+FlakeRef FlakeRef::resolve(ref<Store> store) const
+{
+ auto [input2, extraAttrs] = lookupInRegistries(store, input);
+ return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
+}
+
+FlakeRef parseFlakeRef(
+ const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+{
+ auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
+ if (fragment != "")
+ throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
+ return flakeRef;
+}
+
+std::optional<FlakeRef> maybeParseFlakeRef(
+ const std::string & url, const std::optional<Path> & baseDir)
+{
+ try {
+ return parseFlakeRef(url, baseDir);
+ } catch (Error &) {
+ return {};
+ }
+}
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+ const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
+{
+ using namespace fetchers;
+
+ static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
+
+ static std::regex pathUrlRegex(
+ "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
+ + "(?:\\?(" + queryRegex + "))?"
+ + "(?:#(" + queryRegex + "))?",
+ std::regex::ECMAScript);
+
+ static std::regex flakeRegex(
+ "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+ + "(?:#(" + queryRegex + "))?",
+ std::regex::ECMAScript);
+
+ std::smatch match;
+
+ /* Check if 'url' is a flake ID. This is an abbreviated syntax for
+ 'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
+
+ if (std::regex_match(url, match, flakeRegex)) {
+ auto parsedURL = ParsedURL{
+ .url = url,
+ .base = "flake:" + std::string(match[1]),
+ .scheme = "flake",
+ .authority = "",
+ .path = match[1],
+ };
+
+ return std::make_pair(
+ FlakeRef(Input::fromURL(parsedURL), ""),
+ percentDecode(std::string(match[6])));
+ }
+
+ else if (std::regex_match(url, match, pathUrlRegex)) {
+ std::string path = match[1];
+ std::string fragment = percentDecode(std::string(match[3]));
+
+ if (baseDir) {
+ /* Check if 'url' is a path (either absolute or relative
+ to 'baseDir'). If so, search upward to the root of the
+ repo (i.e. the directory containing .git). */
+
+ path = absPath(path, baseDir, true);
+
+ if (!S_ISDIR(lstat(path).st_mode))
+ throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
+
+ if (!allowMissing && !pathExists(path + "/flake.nix"))
+ throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
+
+ auto flakeRoot = path;
+ std::string subdir;
+
+ while (flakeRoot != "/") {
+ if (pathExists(flakeRoot + "/.git")) {
+ auto base = std::string("git+file://") + flakeRoot;
+
+ auto parsedURL = ParsedURL{
+ .url = base, // FIXME
+ .base = base,
+ .scheme = "git+file",
+ .authority = "",
+ .path = flakeRoot,
+ .query = decodeQuery(match[2]),
+ };
+
+ if (subdir != "") {
+ if (parsedURL.query.count("dir"))
+ throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
+ parsedURL.query.insert_or_assign("dir", subdir);
+ }
+
+ if (pathExists(flakeRoot + "/.git/shallow"))
+ parsedURL.query.insert_or_assign("shallow", "1");
+
+ return std::make_pair(
+ FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+ fragment);
+ }
+
+ subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+ flakeRoot = dirOf(flakeRoot);
+ }
+
+ } else {
+ if (!hasPrefix(path, "/"))
+ throw BadURL("flake reference '%s' is not an absolute path", url);
+ path = canonPath(path);
+ }
+
+ fetchers::Attrs attrs;
+ attrs.insert_or_assign("type", "path");
+ attrs.insert_or_assign("path", path);
+
+ return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
+ }
+
+ else {
+ auto parsedURL = parseURL(url);
+ std::string fragment;
+ std::swap(fragment, parsedURL.fragment);
+ return std::make_pair(
+ FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
+ fragment);
+ }
+}
+
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+ const std::string & url, const std::optional<Path> & baseDir)
+{
+ try {
+ return parseFlakeRefWithFragment(url, baseDir);
+ } catch (Error & e) {
+ return {};
+ }
+}
+
+FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
+{
+ auto attrs2(attrs);
+ attrs2.erase("dir");
+ return FlakeRef(
+ fetchers::Input::fromAttrs(std::move(attrs2)),
+ fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
+}
+
+std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
+{
+ auto [tree, lockedInput] = input.fetch(store);
+ return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
+}
+
+}
diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh
new file mode 100644
index 000000000..f4eb825a6
--- /dev/null
+++ b/src/libexpr/flake/flakeref.hh
@@ -0,0 +1,53 @@
+#pragma once
+
+#include "types.hh"
+#include "hash.hh"
+#include "fetchers.hh"
+
+#include <variant>
+
+namespace nix {
+
+class Store;
+
+typedef std::string FlakeId;
+
+struct FlakeRef
+{
+ fetchers::Input input;
+
+ Path subdir;
+
+ bool operator==(const FlakeRef & other) const;
+
+ FlakeRef(fetchers::Input && input, const Path & subdir)
+ : input(std::move(input)), subdir(subdir)
+ { }
+
+ // FIXME: change to operator <<.
+ std::string to_string() const;
+
+ fetchers::Attrs toAttrs() const;
+
+ FlakeRef resolve(ref<Store> store) const;
+
+ static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
+
+ std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
+};
+
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
+
+FlakeRef parseFlakeRef(
+ const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+
+std::optional<FlakeRef> maybeParseFlake(
+ const std::string & url, const std::optional<Path> & baseDir = {});
+
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+ const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
+
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+ const std::string & url, const std::optional<Path> & baseDir = {});
+
+}
diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
new file mode 100644
index 000000000..a74846944
--- /dev/null
+++ b/src/libexpr/flake/lockfile.cc
@@ -0,0 +1,338 @@
+#include "lockfile.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::flake {
+
+FlakeRef getFlakeRef(
+ const nlohmann::json & json,
+ const char * attr,
+ const char * info)
+{
+ auto i = json.find(attr);
+ if (i != json.end()) {
+ auto attrs = jsonToAttrs(*i);
+ // FIXME: remove when we drop support for version 5.
+ if (info) {
+ auto j = json.find(info);
+ if (j != json.end()) {
+ for (auto k : jsonToAttrs(*j))
+ attrs.insert_or_assign(k.first, k.second);
+ }
+ }
+ return FlakeRef::fromAttrs(attrs);
+ }
+
+ throw Error("attribute '%s' missing in lock file", attr);
+}
+
+LockedNode::LockedNode(const nlohmann::json & json)
+ : lockedRef(getFlakeRef(json, "locked", "info"))
+ , originalRef(getFlakeRef(json, "original", nullptr))
+ , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
+{
+ if (!lockedRef.input.isImmutable())
+ throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
+}
+
+StorePath LockedNode::computeStorePath(Store & store) const
+{
+ return lockedRef.input.computeStorePath(store);
+}
+
+std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
+{
+ auto pos = root;
+
+ if (!pos) return {};
+
+ for (auto & elem : path) {
+ if (auto i = get(pos->inputs, elem)) {
+ if (auto node = std::get_if<0>(&*i))
+ pos = *node;
+ else if (auto follows = std::get_if<1>(&*i)) {
+ pos = findInput(*follows);
+ if (!pos) return {};
+ }
+ } else
+ return {};
+ }
+
+ return pos;
+}
+
+LockFile::LockFile(const nlohmann::json & json, const Path & path)
+{
+ auto version = json.value("version", 0);
+ if (version < 5 || version > 7)
+ throw Error("lock file '%s' has unsupported version %d", path, version);
+
+ std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
+
+ std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
+
+ getInputs = [&](Node & node, const nlohmann::json & jsonNode)
+ {
+ if (jsonNode.find("inputs") == jsonNode.end()) return;
+ for (auto & i : jsonNode["inputs"].items()) {
+ if (i.value().is_array()) {
+ InputPath path;
+ for (auto & j : i.value())
+ path.push_back(j);
+ node.inputs.insert_or_assign(i.key(), path);
+ } else {
+ std::string inputKey = i.value();
+ auto k = nodeMap.find(inputKey);
+ if (k == nodeMap.end()) {
+ auto jsonNode2 = json["nodes"][inputKey];
+ auto input = std::make_shared<LockedNode>(jsonNode2);
+ k = nodeMap.insert_or_assign(inputKey, input).first;
+ getInputs(*input, jsonNode2);
+ }
+ if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
+ node.inputs.insert_or_assign(i.key(), child);
+ else
+ // FIXME: replace by follows node
+ throw Error("lock file contains cycle to root node");
+ }
+ }
+ };
+
+ std::string rootKey = json["root"];
+ nodeMap.insert_or_assign(rootKey, root);
+ getInputs(*root, json["nodes"][rootKey]);
+
+ // FIXME: check that there are no cycles in version >= 7. Cycles
+ // between inputs are only possible using 'follows' indirections.
+ // Once we drop support for version <= 6, we can simplify the code
+ // a bit since we don't need to worry about cycles.
+}
+
+nlohmann::json LockFile::toJson() const
+{
+ nlohmann::json nodes;
+ std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
+ std::unordered_set<std::string> keys;
+
+ std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode;
+
+ dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string
+ {
+ auto k = nodeKeys.find(node);
+ if (k != nodeKeys.end())
+ return k->second;
+
+ if (!keys.insert(key).second) {
+ for (int n = 2; ; ++n) {
+ auto k = fmt("%s_%d", key, n);
+ if (keys.insert(k).second) {
+ key = k;
+ break;
+ }
+ }
+ }
+
+ nodeKeys.insert_or_assign(node, key);
+
+ auto n = nlohmann::json::object();
+
+ if (!node->inputs.empty()) {
+ auto inputs = nlohmann::json::object();
+ for (auto & i : node->inputs) {
+ if (auto child = std::get_if<0>(&i.second)) {
+ inputs[i.first] = dumpNode(i.first, *child);
+ } else if (auto follows = std::get_if<1>(&i.second)) {
+ auto arr = nlohmann::json::array();
+ for (auto & x : *follows)
+ arr.push_back(x);
+ inputs[i.first] = std::move(arr);
+ }
+ }
+ n["inputs"] = std::move(inputs);
+ }
+
+ if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
+ n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
+ n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
+ if (!lockedNode->isFlake) n["flake"] = false;
+ }
+
+ nodes[key] = std::move(n);
+
+ return key;
+ };
+
+ nlohmann::json json;
+ json["version"] = 7;
+ json["root"] = dumpNode("root", root);
+ json["nodes"] = std::move(nodes);
+
+ return json;
+}
+
+std::string LockFile::to_string() const
+{
+ return toJson().dump(2);
+}
+
+LockFile LockFile::read(const Path & path)
+{
+ if (!pathExists(path)) return LockFile();
+ return LockFile(nlohmann::json::parse(readFile(path)), path);
+}
+
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
+{
+ stream << lockFile.toJson().dump(2);
+ return stream;
+}
+
+void LockFile::write(const Path & path) const
+{
+ createDirs(dirOf(path));
+ writeFile(path, fmt("%s\n", *this));
+}
+
+bool LockFile::isImmutable() const
+{
+ std::unordered_set<std::shared_ptr<const Node>> nodes;
+
+ std::function<void(std::shared_ptr<const Node> node)> visit;
+
+ visit = [&](std::shared_ptr<const Node> node)
+ {
+ if (!nodes.insert(node).second) return;
+ for (auto & i : node->inputs)
+ if (auto child = std::get_if<0>(&i.second))
+ visit(*child);
+ };
+
+ visit(root);
+
+ for (auto & i : nodes) {
+ if (i == root) continue;
+ auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
+ if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
+ }
+
+ return true;
+}
+
+bool LockFile::operator ==(const LockFile & other) const
+{
+ // FIXME: slow
+ return toJson() == other.toJson();
+}
+
+InputPath parseInputPath(std::string_view s)
+{
+ InputPath path;
+
+ for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
+ if (!std::regex_match(elem, flakeIdRegex))
+ throw UsageError("invalid flake input path element '%s'", elem);
+ path.push_back(elem);
+ }
+
+ return path;
+}
+
+std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
+{
+ std::unordered_set<std::shared_ptr<Node>> done;
+ std::map<InputPath, Node::Edge> res;
+
+ std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse;
+
+ recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node)
+ {
+ if (!done.insert(node).second) return;
+
+ for (auto &[id, input] : node->inputs) {
+ auto inputPath(prefix);
+ inputPath.push_back(id);
+ res.emplace(inputPath, input);
+ if (auto child = std::get_if<0>(&input))
+ recurse(inputPath, *child);
+ }
+ };
+
+ recurse({}, root);
+
+ return res;
+}
+
+std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
+{
+ if (auto node = std::get_if<0>(&edge))
+ stream << "'" << (*node)->lockedRef << "'";
+ else if (auto follows = std::get_if<1>(&edge))
+ stream << fmt("follows '%s'", printInputPath(*follows));
+ return stream;
+}
+
+static bool equals(const Node::Edge & e1, const Node::Edge & e2)
+{
+ if (auto n1 = std::get_if<0>(&e1))
+ if (auto n2 = std::get_if<0>(&e2))
+ return (*n1)->lockedRef == (*n2)->lockedRef;
+ if (auto f1 = std::get_if<1>(&e1))
+ if (auto f2 = std::get_if<1>(&e2))
+ return *f1 == *f2;
+ return false;
+}
+
+std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
+{
+ auto oldFlat = oldLocks.getAllInputs();
+ auto newFlat = newLocks.getAllInputs();
+
+ auto i = oldFlat.begin();
+ auto j = newFlat.begin();
+ std::string res;
+
+ while (i != oldFlat.end() || j != newFlat.end()) {
+ if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
+ res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
+ ++j;
+ } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
+ res += fmt("* Removed '%s'\n", printInputPath(i->first));
+ ++i;
+ } else {
+ if (!equals(i->second, j->second)) {
+ res += fmt("* Updated '%s': %s -> %s\n",
+ printInputPath(i->first),
+ i->second,
+ j->second);
+ }
+ ++i;
+ ++j;
+ }
+ }
+
+ return res;
+}
+
+void LockFile::check()
+{
+ auto inputs = getAllInputs();
+
+ for (auto & [inputPath, input] : inputs) {
+ if (auto follows = std::get_if<1>(&input)) {
+ if (!follows->empty() && !get(inputs, *follows))
+ throw Error("input '%s' follows a non-existent input '%s'",
+ printInputPath(inputPath),
+ printInputPath(*follows));
+ }
+ }
+}
+
+void check();
+
+std::string printInputPath(const InputPath & path)
+{
+ return concatStringsSep("/", path);
+}
+
+}
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
new file mode 100644
index 000000000..5e7cfda3e
--- /dev/null
+++ b/src/libexpr/flake/lockfile.hh
@@ -0,0 +1,85 @@
+#pragma once
+
+#include "flakeref.hh"
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+class Store;
+struct StorePath;
+}
+
+namespace nix::flake {
+
+using namespace fetchers;
+
+typedef std::vector<FlakeId> InputPath;
+
+struct LockedNode;
+
+/* A node in the lock file. It has outgoing edges to other nodes (its
+ inputs). Only the root node has this type; all other nodes have
+ type LockedNode. */
+struct Node : std::enable_shared_from_this<Node>
+{
+ typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge;
+
+ std::map<FlakeId, Edge> inputs;
+
+ virtual ~Node() { }
+};
+
+/* A non-root node in the lock file. */
+struct LockedNode : Node
+{
+ FlakeRef lockedRef, originalRef;
+ bool isFlake = true;
+
+ LockedNode(
+ const FlakeRef & lockedRef,
+ const FlakeRef & originalRef,
+ bool isFlake = true)
+ : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
+ { }
+
+ LockedNode(const nlohmann::json & json);
+
+ StorePath computeStorePath(Store & store) const;
+};
+
+struct LockFile
+{
+ std::shared_ptr<Node> root = std::make_shared<Node>();
+
+ LockFile() {};
+ LockFile(const nlohmann::json & json, const Path & path);
+
+ nlohmann::json toJson() const;
+
+ std::string to_string() const;
+
+ static LockFile read(const Path & path);
+
+ void write(const Path & path) const;
+
+ bool isImmutable() const;
+
+ bool operator ==(const LockFile & other) const;
+
+ std::shared_ptr<Node> findInput(const InputPath & path);
+
+ std::map<InputPath, Node::Edge> getAllInputs() const;
+
+ static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
+
+ /* Check that every 'follows' input target exists. */
+ void check();
+};
+
+std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
+
+InputPath parseInputPath(std::string_view s);
+
+std::string printInputPath(const InputPath & path);
+
+}
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 9055f59a1..5d6e39aa0 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
- outPath = store->printStorePath(i->second.path);
+ outPath = store->printStorePath(i->second.path(*store, drv.name));
}
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 9ed39e745..d84b150e0 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr
libexpr_DIR := $(d)
-libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
+libexpr_SOURCES := \
+ $(wildcard $(d)/*.cc) \
+ $(wildcard $(d)/primops/*.cc) \
+ $(wildcard $(d)/flake/*.cc) \
+ $(d)/lexer-tab.cc \
+ $(d)/parser-tab.cc
libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
@@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
+$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
+ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
+
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
+
+$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index b4b65883d..d5698011f 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
if (!pos)
str << "undefined position";
else
- str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
+ {
+ auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
+ switch (pos.origin) {
+ case foFile:
+ f % (string) pos.file;
+ break;
+ case foStdin:
+ case foString:
+ f % "(string)";
+ break;
+ default:
+ throw Error("unhandled Pos origin!");
+ }
+ str << (f % pos.line % pos.column).str();
+ }
+
return str;
}
@@ -270,7 +285,7 @@ void ExprVar::bindVars(const StaticEnv & env)
if (withLevel == -1)
throw UndefinedVarError({
.hint = hintfmt("undefined variable '%1%'", name),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
fromWith = true;
this->level = withLevel;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index ec6fd3190..e4cbc660f 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -24,11 +24,12 @@ MakeError(RestrictedPathError, Error);
struct Pos
{
+ FileOrigin origin;
Symbol file;
unsigned int line, column;
- Pos() : line(0), column(0) { };
- Pos(const Symbol & file, unsigned int line, unsigned int column)
- : file(file), line(line), column(column) { };
+ Pos() : origin(foString), line(0), column(0) { };
+ Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
+ : origin(origin), file(file), line(line), column(column) { };
operator bool() const
{
return line != 0;
@@ -238,7 +239,7 @@ struct ExprLambda : Expr
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'", arg),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
};
void setName(Symbol & name);
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index a639be64e..24b21f7da 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -30,7 +30,8 @@ namespace nix {
SymbolTable & symbols;
Expr * result;
Path basePath;
- Symbol path;
+ Symbol file;
+ FileOrigin origin;
ErrorInfo error;
Symbol sLetBody;
ParseData(EvalState & state)
@@ -65,18 +66,17 @@ namespace nix {
static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
{
throw ParseError({
- .hint = hintfmt("attribute '%1%' already defined at %2%",
+ .hint = hintfmt("attribute '%1%' already defined at %2%",
showAttrPath(attrPath), prevPos),
- .nixCode = NixCode { .errPos = pos },
+ .errPos = pos
});
}
-
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
{
throw ParseError({
.hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
- .nixCode = NixCode { .errPos = pos },
+ .errPos = pos
});
}
@@ -148,7 +148,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
throw ParseError({
.hint = hintfmt("duplicate formal function argument '%1%'",
formal.name),
- .nixCode = NixCode { .errPos = pos },
+ .errPos = pos
});
formals->formals.push_front(formal);
}
@@ -246,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
{
- return Pos(data->path, loc.first_line, loc.first_column);
+ return Pos(data->origin, data->file, loc.first_line, loc.first_column);
}
#define CUR_POS makeCurPos(*yylocp, data)
@@ -259,7 +259,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
{
data->error = {
.hint = hintfmt(error),
- .nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
+ .errPos = makeCurPos(*loc, data)
};
}
@@ -339,7 +339,7 @@ expr_function
{ if (!$2->dynamicAttrs.empty())
throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in let"),
- .nixCode = NixCode { .errPos = CUR_POS },
+ .errPos = CUR_POS
});
$$ = new ExprLet($2, $4);
}
@@ -419,7 +419,7 @@ expr_simple
if (noURLLiterals)
throw ParseError({
.hint = hintfmt("URL literals are disabled"),
- .nixCode = NixCode { .errPos = CUR_POS }
+ .errPos = CUR_POS
});
$$ = new ExprString(data->symbols.create($1));
}
@@ -492,7 +492,7 @@ attrs
} else
throw ParseError({
.hint = hintfmt("dynamic attributes not allowed in inherit"),
- .nixCode = NixCode { .errPos = makeCurPos(@2, data) },
+ .errPos = makeCurPos(@2, data)
});
}
| { $$ = new AttrPath; }
@@ -569,13 +569,24 @@ formal
namespace nix {
-Expr * EvalState::parse(const char * text,
+Expr * EvalState::parse(const char * text, FileOrigin origin,
const Path & path, const Path & basePath, StaticEnv & staticEnv)
{
yyscan_t scanner;
ParseData data(*this);
+ data.origin = origin;
+ switch (origin) {
+ case foFile:
+ data.file = data.symbols.create(path);
+ break;
+ case foStdin:
+ case foString:
+ data.file = data.symbols.create(text);
+ break;
+ default:
+ assert(false);
+ }
data.basePath = basePath;
- data.path = data.symbols.create(path);
yylex_init(&scanner);
yy_scan_string(text, scanner);
@@ -625,13 +636,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
{
- return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
+ return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
}
Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
{
- return parse(s.data(), "(string)", basePath, staticEnv);
+ return parse(s.data(), foString, "", basePath, staticEnv);
}
@@ -644,7 +655,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
Expr * EvalState::parseStdin()
{
//Activity act(*logger, lvlTalkative, format("parsing standard input"));
- return parseExprFromString(drainFD(0), absPath("."));
+ return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
}
@@ -693,7 +704,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -708,7 +719,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
if (isUri(elem.second)) {
try {
res = { true, store->toRealPath(fetchers::downloadTarball(
- store, resolveUri(elem.second), "source", false).storePath) };
+ store, resolveUri(elem.second), "source", false).first.storePath) };
} catch (FileTransferError & e) {
logWarning({
.name = "Entry download",
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 3830d8107..05d499d1f 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -30,18 +30,6 @@ namespace nix {
*************************************************************/
-/* Decode a context string ‘!<name>!<path>’ into a pair <path,
- name>. */
-std::pair<string, string> decodeContext(const string & s)
-{
- if (s.at(0) == '!') {
- size_t index = s.find("!", 1);
- return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
- } else
- return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
-}
-
-
InvalidPathError::InvalidPathError(const Path & path) :
EvalError("path '%s' is not valid", path), path(path) {}
@@ -64,7 +52,7 @@ void EvalState::realiseContext(const PathSet & context)
DerivationOutputs::iterator i = drv.outputs.find(outputName);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
- allowedPaths->insert(store->printStorePath(i->second.path));
+ allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
}
}
}
@@ -77,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context)
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
store->buildPaths(drvs);
@@ -96,15 +84,24 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
} catch (InvalidPathError & e) {
throw EvalError({
.hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
Path realPath = state.checkSourcePath(state.toRealPath(path, context));
// FIXME
- if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
- Derivation drv = readDerivation(*state.store, realPath);
+ auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
+ if (!state.store->isStorePath(path))
+ return std::nullopt;
+ auto storePath = state.store->parseStorePath(path);
+ if (!(state.store->isValidPath(storePath) && isDerivation(path)))
+ return std::nullopt;
+ return storePath;
+ };
+ if (auto optStorePath = isValidDerivationInStore()) {
+ auto storePath = *optStorePath;
+ Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath));
Value & w = *state.allocValue();
state.mkAttrs(w, 3 + drv.outputs.size());
Value * v2 = state.allocAttr(w, state.sDrvPath);
@@ -118,7 +115,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
for (const auto & o : drv.outputs) {
v2 = state.allocAttr(w, state.symbols.create(o.first));
- mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path});
+ mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
outputsVal->listElems()[outputs_index] = state.allocValue();
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
}
@@ -177,7 +174,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
.hint = hintfmt(
"cannot import '%1%', since path '%2%' is not valid",
path, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -215,7 +212,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
if (count == 0) {
throw EvalError({
.hint = hintfmt("at least one argument to 'exec' required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
PathSet context;
@@ -230,7 +227,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
throw EvalError({
.hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
program, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -239,13 +236,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
try {
parsed = state.parseExprFromString(output, pos.file);
} catch (Error & e) {
- e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
+ e.addTrace(pos, "While parsing the output from '%1%'", program);
throw;
}
try {
state.eval(parsed, v);
} catch (Error & e) {
- e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
+ e.addTrace(pos, "While evaluating the output from '%1%'", program);
throw;
}
}
@@ -385,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
if (startSet == args[0]->attrs->end())
throw EvalError({
.hint = hintfmt("attribute 'startSet' required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.forceList(*startSet->value, pos);
@@ -399,7 +396,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
if (op == args[0]->attrs->end())
throw EvalError({
.hint = hintfmt("attribute 'operator' required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.forceValue(*op->value, pos);
@@ -421,7 +418,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
if (key == e->attrs->end())
throw EvalError({
.hint = hintfmt("attribute 'key' required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.forceValue(*key->value, pos);
@@ -471,7 +468,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
v = *args[1];
} catch (Error & e) {
PathSet context;
- e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
+ e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
throw;
}
}
@@ -556,14 +553,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (attr == args[0]->attrs->end())
throw EvalError({
.hint = hintfmt("required attribute 'name' missing"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
string drvName;
Pos & posDrvName(*attr->pos);
try {
drvName = state.forceStringNoCtx(*attr->value, pos);
} catch (Error & e) {
- e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
+ e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
throw;
}
@@ -582,6 +579,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Build the derivation expression by processing the attributes. */
Derivation drv;
+ drv.name = drvName;
PathSet context;
@@ -603,7 +601,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
else
throw EvalError({
.hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
};
@@ -613,7 +611,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (outputs.find(j) != outputs.end())
throw EvalError({
.hint = hintfmt("duplicate derivation output '%1%'", j),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
/* !!! Check whether j is a valid attribute
name. */
@@ -623,14 +621,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (j == "drv")
throw EvalError({
.hint = hintfmt("invalid derivation output name 'drv'" ),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
outputs.insert(j);
}
if (outputs.empty())
throw EvalError({
.hint = hintfmt("derivation cannot have an empty set of outputs"),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
};
@@ -696,8 +694,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
}
} catch (Error & e) {
- e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
- % key % drvName % posDrvName);
+ e.addTrace(posDrvName,
+ "while evaluating the attribute '%1%' of the derivation '%2%'",
+ key, drvName);
throw;
}
}
@@ -745,20 +744,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (drv.builder == "")
throw EvalError({
.hint = hintfmt("required attribute 'builder' missing"),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
if (drv.platform == "")
throw EvalError({
.hint = hintfmt("required attribute 'system' missing"),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
/* Check whether the derivation name is valid. */
if (isDerivation(drvName))
throw EvalError({
.hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
if (outputHash) {
@@ -766,7 +765,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error({
.hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
- .nixCode = NixCode { .errPos = posDrvName }
+ .errPos = posDrvName
});
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
@@ -775,11 +774,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign("out", DerivationOutput {
- .path = std::move(outPath),
- .hash = FixedOutputHash {
- .method = ingestionMethod,
- .hash = std::move(h),
- },
+ .output = DerivationOutputFixed {
+ .hash = FixedOutputHash {
+ .method = ingestionMethod,
+ .hash = std::move(h),
+ },
+ },
});
}
@@ -794,8 +794,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env[i] = "";
drv.outputs.insert_or_assign(i,
DerivationOutput {
- .path = StorePath::dummy,
- .hash = std::optional<FixedOutputHash> {},
+ .output = DerivationOutputInputAddressed {
+ .path = StorePath::dummy,
+ },
});
}
@@ -806,8 +807,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(i,
DerivationOutput {
- .path = std::move(outPath),
- .hash = std::optional<FixedOutputHash>(),
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(outPath),
+ },
});
}
}
@@ -828,7 +830,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
for (auto & i : drv.outputs) {
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
- state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
+ state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
}
v.attrs->sort();
}
@@ -880,12 +882,12 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
if (!state.store->isInStore(path))
throw EvalError({
.hint = hintfmt("path '%1%' is not in the Nix store", path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
- Path path2 = state.store->toStorePath(path);
+ auto path2 = state.store->toStorePath(path).first;
if (!settings.readOnlyMode)
- state.store->ensurePath(state.store->parseStorePath(path2));
- context.insert(path2);
+ state.store->ensurePath(path2);
+ context.insert(state.store->printStorePath(path2));
mkString(v, path, context);
}
@@ -901,7 +903,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
.hint = hintfmt(
"cannot check the existence of '%1%', since path '%2%' is not valid",
path, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -947,7 +949,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
} catch (InvalidPathError & e) {
throw EvalError({
.hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
@@ -978,7 +980,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
if (i == v2.attrs->end())
throw EvalError({
.hint = hintfmt("attribute 'path' missing"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
PathSet context;
@@ -989,7 +991,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
} catch (InvalidPathError & e) {
throw EvalError({
.hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -1009,7 +1011,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
if (!ht)
throw Error({
.hint = hintfmt("unknown hash type '%1%'", type),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
PathSet context; // discarded
@@ -1028,7 +1030,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
} catch (InvalidPathError & e) {
throw EvalError({
.hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
@@ -1104,7 +1106,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
"in 'toFile': the file named '%1%' must not contain a reference "
"to a derivation but contains (%2%)",
name, path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
refs.insert(state.store->parseStorePath(path));
}
@@ -1122,7 +1124,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
- Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
+ Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
{
const auto path = evalSettings.pureEval && expectedHash ?
path_ :
@@ -1153,7 +1155,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
std::optional<StorePath> expectedStorePath;
if (expectedHash)
- expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
+ expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
Path dstPath;
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
dstPath = state.store->printStorePath(settings.readOnlyMode
@@ -1175,7 +1177,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
if (!context.empty())
throw EvalError({
.hint = hintfmt("string '%1%' cannot refer to other paths", path),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.forceValue(*args[0], pos);
@@ -1184,10 +1186,10 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.hint = hintfmt(
"first argument in call to 'filterSource' is not a function but %1%",
showType(*args[0])),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
- addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
+ addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
}
static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
@@ -1197,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
string name;
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
- Hash expectedHash;
+ std::optional<Hash> expectedHash;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
@@ -1207,7 +1209,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
if (!context.empty())
throw EvalError({
.hint = hintfmt("string '%1%' cannot refer to other paths", path),
- .nixCode = NixCode { .errPos = *attr.pos }
+ .errPos = *attr.pos
});
} else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos);
@@ -1221,13 +1223,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
else
throw EvalError({
.hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
- .nixCode = NixCode { .errPos = *attr.pos }
+ .errPos = *attr.pos
});
}
if (path.empty())
throw EvalError({
.hint = hintfmt("'path' required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
if (name.empty())
name = baseNameOf(path);
@@ -1288,7 +1290,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
if (i == args[1]->attrs->end())
throw EvalError({
.hint = hintfmt("attribute '%1%' missing", attr),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
// !!! add to stack trace?
if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
@@ -1371,7 +1373,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
if (j == v2.attrs->end())
throw TypeError({
.hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
string name = state.forceStringNoCtx(*j->value, pos);
@@ -1381,7 +1383,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
if (j2 == v2.attrs->end())
throw TypeError({
.hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
v.attrs->push_back(Attr(sym, j2->value, j2->pos));
}
@@ -1457,7 +1459,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
if (args[0]->type != tLambda)
throw TypeError({
.hint = hintfmt("'functionArgs' requires a function"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
if (!args[0]->lambda.fun->matchAttrs) {
@@ -1513,7 +1515,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
if (n < 0 || (unsigned int) n >= list.listSize())
throw Error({
.hint = hintfmt("list index %1% is out of bounds", n),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.forceValue(*list.listElems()[n], pos);
v = *list.listElems()[n];
@@ -1543,7 +1545,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
if (args[0]->listSize() == 0)
throw Error({
.hint = hintfmt("'tail' called on an empty list"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.mkList(v, args[0]->listSize() - 1);
@@ -1688,7 +1690,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
if (len < 0)
throw EvalError({
.hint = hintfmt("cannot create list of size %1%", len),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
state.mkList(v, len);
@@ -1850,7 +1852,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
if (f2 == 0)
throw EvalError({
.hint = hintfmt("division by zero"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
if (args[0]->type == tFloat || args[1]->type == tFloat) {
@@ -1862,7 +1864,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
throw EvalError({
.hint = hintfmt("overflow in integer division"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
mkInt(v, i1 / i2);
@@ -1923,7 +1925,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
if (start < 0)
throw EvalError({
.hint = hintfmt("negative start position in 'substring'"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
@@ -1946,7 +1948,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
if (!ht)
throw Error({
.hint = hintfmt("unknown hash type '%1%'", type),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
PathSet context; // discarded
@@ -1992,12 +1994,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
throw EvalError({
.hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
} else {
throw EvalError({
.hint = hintfmt("invalid regular expression '%s'", re),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
}
@@ -2065,12 +2067,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
throw EvalError({
.hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
} else {
throw EvalError({
.hint = hintfmt("invalid regular expression '%s'", re),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
}
@@ -2104,7 +2106,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
if (args[0]->listSize() != args[1]->listSize())
throw EvalError({
.hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
vector<string> from;
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 301e8c5dd..dbb93bae6 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (!state.store->isStorePath(i.name))
throw EvalError({
.hint = hintfmt("Context key '%s' is not a store path", i.name),
- .nixCode = NixCode { .errPos = *i.pos }
+ .errPos = *i.pos
});
if (!settings.readOnlyMode)
state.store->ensurePath(state.store->parseStorePath(i.name));
@@ -165,7 +165,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (!isDerivation(i.name)) {
throw EvalError({
.hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
- .nixCode = NixCode { .errPos = *i.pos }
+ .errPos = *i.pos
});
}
context.insert("=" + string(i.name));
@@ -178,7 +178,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError({
.hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
- .nixCode = NixCode { .errPos = *i.pos }
+ .errPos = *i.pos
});
}
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc
deleted file mode 100644
index dd7229a3d..000000000
--- a/src/libexpr/primops/fetchGit.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-#include "primops.hh"
-#include "eval-inline.hh"
-#include "store-api.hh"
-#include "hash.hh"
-#include "fetchers.hh"
-#include "url.hh"
-
-namespace nix {
-
-static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
-{
- std::string url;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
- std::string name = "source";
- bool fetchSubmodules = false;
- PathSet context;
-
- state.forceValue(*args[0]);
-
- if (args[0]->type == tAttrs) {
-
- state.forceAttrs(*args[0], pos);
-
- for (auto & attr : *args[0]->attrs) {
- string n(attr.name);
- if (n == "url")
- url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
- else if (n == "ref")
- ref = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "rev")
- rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
- else if (n == "name")
- name = state.forceStringNoCtx(*attr.value, *attr.pos);
- else if (n == "submodules")
- fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
- else
- throw EvalError({
- .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
- .nixCode = NixCode { .errPos = *attr.pos }
- });
- }
-
- if (url.empty())
- throw EvalError({
- .hint = hintfmt("'url' argument required"),
- .nixCode = NixCode { .errPos = pos }
- });
-
- } else
- url = state.coerceToString(pos, *args[0], context, false, false);
-
- // FIXME: git externals probably can be used to bypass the URI
- // whitelist. Ah well.
- state.checkURI(url);
-
- if (evalSettings.pureEval && !rev)
- throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
-
- fetchers::Attrs attrs;
- attrs.insert_or_assign("type", "git");
- attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
- if (ref) attrs.insert_or_assign("ref", *ref);
- if (rev) attrs.insert_or_assign("rev", rev->gitRev());
- if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
- auto input = fetchers::inputFromAttrs(attrs);
-
- // FIXME: use name?
- auto [tree, input2] = input->fetchTree(state.store);
-
- state.mkAttrs(v, 8);
- auto storePath = state.store->printStorePath(tree.storePath);
- mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
- // Backward compatibility: set 'rev' to
- // 0000000000000000000000000000000000000000 for a dirty tree.
- auto rev2 = input2->getRev().value_or(Hash(htSHA1));
- mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
- mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
- // Backward compatibility: set 'revCount' to 0 for a dirty tree.
- mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
- tree.info.revCount.value_or(0));
- mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
- v.attrs->sort();
-
- if (state.allowedPaths)
- state.allowedPaths->insert(tree.actualPath);
-}
-
-static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
-
-}
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 9bace8f89..fc2a6a1c2 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -40,14 +40,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
- .nixCode = NixCode { .errPos = *attr.pos }
+ .errPos = *attr.pos
});
}
if (url.empty())
throw EvalError({
.hint = hintfmt("'url' argument required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
} else
@@ -65,23 +65,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
if (ref) attrs.insert_or_assign("ref", *ref);
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
- auto input = fetchers::inputFromAttrs(attrs);
+ auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name
- auto [tree, input2] = input->fetchTree(state.store);
+ auto [tree, input2] = input.fetch(state.store);
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
- if (input2->getRef())
- mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
+ if (input2.getRef())
+ mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
// Backward compatibility: set 'rev' to
// 0000000000000000000000000000000000000000 for a dirty tree.
- auto rev2 = input2->getRev().value_or(Hash(htSHA1));
+ auto rev2 = input2.getRev().value_or(Hash(htSHA1));
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
- if (tree.info.revCount)
- mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+ if (auto revCount = input2.getRevCount())
+ mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort();
if (state.allowedPaths)
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 9be93710a..cddcf0e59 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -3,6 +3,7 @@
#include "store-api.hh"
#include "fetchers.hh"
#include "filetransfer.hh"
+#include "registry.hh"
#include <ctime>
#include <iomanip>
@@ -12,39 +13,73 @@ namespace nix {
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
- std::shared_ptr<const fetchers::Input> input,
- Value & v)
+ const fetchers::Input & input,
+ Value & v,
+ bool emptyRevFallback)
{
+ assert(input.isImmutable());
+
state.mkAttrs(v, 8);
auto storePath = state.store->printStorePath(tree.storePath);
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
- assert(tree.info.narHash);
- mkString(*state.allocAttr(v, state.symbols.create("narHash")),
- tree.info.narHash.to_string(SRI, true));
+ // FIXME: support arbitrary input attributes.
- if (input->getRev()) {
- mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
- mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
+ auto narHash = input.getNarHash();
+ assert(narHash);
+ mkString(*state.allocAttr(v, state.symbols.create("narHash")),
+ narHash->to_string(SRI, true));
+
+ if (auto rev = input.getRev()) {
+ mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
+ mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
+ } else if (emptyRevFallback) {
+ // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
+ auto emptyHash = Hash(htSHA1);
+ mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
+ mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
}
- if (tree.info.revCount)
- mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
+ if (input.getType() == "git")
+ mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
- if (tree.info.lastModified)
- mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
- fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
+ if (auto revCount = input.getRevCount())
+ mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
+ else if (emptyRevFallback)
+ mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
+
+ if (auto lastModified = input.getLastModified()) {
+ mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
+ mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
+ fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
+ }
v.attrs->sort();
}
-static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+std::string fixURI(std::string uri, EvalState &state)
{
- settings.requireExperimentalFeature("flakes");
+ state.checkURI(uri);
+ return uri.find("://") != std::string::npos ? uri : "file://" + uri;
+}
+
+void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
+{
+ string n(name);
+ attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
+}
- std::shared_ptr<const fetchers::Input> input;
+static void fetchTree(
+ EvalState &state,
+ const Pos &pos,
+ Value **args,
+ Value &v,
+ const std::optional<std::string> type,
+ bool emptyRevFallback = false
+) {
+ fetchers::Input input;
PathSet context;
state.forceValue(*args[0]);
@@ -56,35 +91,65 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
- if (attr.value->type == tString)
- attrs.emplace(attr.name, attr.value->string.s);
+ if (attr.value->type == tPath || attr.value->type == tString)
+ addURI(
+ state,
+ attrs,
+ attr.name,
+ state.coerceToString(*attr.pos, *attr.value, context, false, false)
+ );
+ else if (attr.value->type == tString)
+ addURI(state, attrs, attr.name, attr.value->string.s);
else if (attr.value->type == tBool)
- attrs.emplace(attr.name, attr.value->boolean);
+ attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
+ else if (attr.value->type == tInt)
+ attrs.emplace(attr.name, attr.value->integer);
else
- throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
+ throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
attr.name, showType(*attr.value));
}
+ if (type)
+ attrs.emplace("type", type.value());
+
if (!attrs.count("type"))
throw Error({
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
- input = fetchers::inputFromAttrs(attrs);
- } else
- input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
+ input = fetchers::Input::fromAttrs(std::move(attrs));
+ } else {
+ auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
+
+ if (type == "git") {
+ fetchers::Attrs attrs;
+ attrs.emplace("type", "git");
+ attrs.emplace("url", url);
+ input = fetchers::Input::fromAttrs(std::move(attrs));
+ } else {
+ input = fetchers::Input::fromURL(url);
+ }
+ }
- if (evalSettings.pureEval && !input->isImmutable())
- throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
+ if (!evalSettings.pureEval && !input.isDirect())
+ input = lookupInRegistries(state.store, input).first;
- // FIXME: use fetchOrSubstituteTree
- auto [tree, input2] = input->fetchTree(state.store);
+ if (evalSettings.pureEval && !input.isImmutable())
+ throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
+
+ auto [tree, input2] = input.fetch(state.store);
if (state.allowedPaths)
state.allowedPaths->insert(tree.actualPath);
- emitTreeAttrs(state, tree, input2, v);
+ emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
+}
+
+static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ settings.requireExperimentalFeature("flakes");
+ fetchTree(state, pos, args, v, std::nullopt);
}
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
@@ -112,14 +177,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
else
throw EvalError({
.hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
- .nixCode = NixCode { .errPos = *attr.pos }
+ .errPos = *attr.pos
});
}
if (!url)
throw EvalError({
.hint = hintfmt("'url' argument required"),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
} else
url = state.forceStringNoCtx(*args[0], pos);
@@ -136,7 +201,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
auto storePath =
unpack
- ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
+ ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
auto path = state.store->toRealPath(storePath);
@@ -147,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
: hashFile(htSHA256, path);
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
- *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
+ *url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
}
if (state.allowedPaths)
@@ -166,7 +231,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args
fetch(state, pos, args, v, "fetchTarball", true, "source");
}
+static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+{
+ fetchTree(state, pos, args, v, "git", true);
+}
+
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
+static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
}
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 7615d1379..b00827a4b 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -83,7 +83,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
} catch (std::runtime_error & e) {
throw EvalError({
.hint = hintfmt("while parsing a TOML string: %s", e.what()),
- .nixCode = NixCode { .errPos = pos }
+ .errPos = pos
});
}
}
diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh
index 7ba5e1c14..4eb6dac81 100644
--- a/src/libexpr/symbol-table.hh
+++ b/src/libexpr/symbol-table.hh
@@ -28,6 +28,12 @@ public:
return s == s2.s;
}
+ // FIXME: remove
+ bool operator == (std::string_view s2) const
+ {
+ return s->compare(s2) == 0;
+ }
+
bool operator != (const Symbol & s2) const
{
return s != s2.s;
@@ -68,9 +74,10 @@ private:
Symbols symbols;
public:
- Symbol create(const string & s)
+ Symbol create(std::string_view s)
{
- std::pair<Symbols::iterator, bool> res = symbols.insert(s);
+ // FIXME: avoid allocation if 's' already exists in the symbol table.
+ std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
return Symbol(&*res.first);
}
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index 71025824e..fe11bb2ed 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -166,6 +166,13 @@ struct Value
{
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
}
+
+ /* Check whether forcing this value requires a trivial amount of
+ computation. In particular, function applications are
+ non-trivial. */
+ bool isTrivial() const;
+
+ std::vector<std::pair<Path, std::string>> getContext();
};
diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index feb0a6085..1e59faa73 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
- if (auto v = std::get_if<int64_t>(&attr.second)) {
+ if (auto v = std::get_if<uint64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
@@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
return *s;
}
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
- if (auto v = std::get_if<int64_t>(&i->second))
+ if (auto v = std::get_if<uint64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not an integer", name);
}
-int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetIntAttr(attrs, name);
if (!s)
@@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
- if (auto v = std::get_if<int64_t>(&i->second))
- return *v;
+ if (auto v = std::get_if<Explicit<bool>>(&i->second))
+ return v->t;
throw Error("input attribute '%s' is not a Boolean", name);
}
@@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
{
std::map<std::string, std::string> query;
for (auto & attr : attrs) {
- if (auto v = std::get_if<int64_t>(&attr.second)) {
+ if (auto v = std::get_if<uint64_t>(&attr.second)) {
query.insert_or_assign(attr.first, fmt("%d", *v));
} else if (auto v = std::get_if<std::string>(&attr.second)) {
query.insert_or_assign(attr.first, *v);
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index d6e0ae000..4b4630c80 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -13,9 +13,14 @@ namespace nix::fetchers {
template<typename T>
struct Explicit {
T t;
+
+ bool operator ==(const Explicit<T> & other) const
+ {
+ return t == other.t;
+ }
};
-typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
@@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
std::string getStrAttr(const Attrs & attrs, const std::string & name);
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
-int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index d76ab1233..3db4f081c 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,6 +6,8 @@ namespace nix::fetchers {
struct Cache
{
+ virtual ~Cache() { }
+
virtual void add(
ref<Store> store,
const Attrs & inAttrs,
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 9174c3de4..28db8aa9c 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -5,71 +5,265 @@
namespace nix::fetchers {
-std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
-void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
{
- if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+ if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme));
}
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+Input Input::fromURL(const std::string & url)
+{
+ return fromURL(parseURL(url));
+}
+
+static void fixupInput(Input & input)
+{
+ // Check common attributes.
+ input.getType();
+ input.getRef();
+ if (input.getRev())
+ input.immutable = true;
+ input.getRevCount();
+ input.getLastModified();
+ if (input.getNarHash())
+ input.immutable = true;
+}
+
+Input Input::fromURL(const ParsedURL & url)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url);
- if (res) return res;
+ if (res) {
+ res->scheme = inputScheme;
+ fixupInput(*res);
+ return std::move(*res);
+ }
}
- throw Error("input '%s' is unsupported", url.url);
-}
-std::unique_ptr<Input> inputFromURL(const std::string & url)
-{
- return inputFromURL(parseURL(url));
+ throw Error("input '%s' is unsupported", url.url);
}
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
+Input Input::fromAttrs(Attrs && attrs)
{
- auto attrs2(attrs);
- attrs2.erase("narHash");
for (auto & inputScheme : *inputSchemes) {
- auto res = inputScheme->inputFromAttrs(attrs2);
+ auto res = inputScheme->inputFromAttrs(attrs);
if (res) {
- if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
- // FIXME: require SRI hash.
- res->narHash = newHashAllowEmpty(*narHash, {});
- return res;
+ res->scheme = inputScheme;
+ fixupInput(*res);
+ return std::move(*res);
}
}
- throw Error("input '%s' is unsupported", attrsToJson(attrs));
+
+ Input input;
+ input.attrs = attrs;
+ fixupInput(input);
+ return input;
+}
+
+ParsedURL Input::toURL() const
+{
+ if (!scheme)
+ throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
+ return scheme->toURL(*this);
+}
+
+std::string Input::to_string() const
+{
+ return toURL().to_string();
}
Attrs Input::toAttrs() const
{
- auto attrs = toAttrsInternal();
- if (narHash)
- attrs.emplace("narHash", narHash->to_string(SRI, true));
- attrs.emplace("type", type());
return attrs;
}
-std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+bool Input::hasAllInfo() const
{
- auto [tree, input] = fetchTreeInternal(store);
+ return getNarHash() && scheme && scheme->hasAllInfo(*this);
+}
+
+bool Input::operator ==(const Input & other) const
+{
+ return attrs == other.attrs;
+}
+
+bool Input::contains(const Input & other) const
+{
+ if (*this == other) return true;
+ auto other2(other);
+ other2.attrs.erase("ref");
+ other2.attrs.erase("rev");
+ if (*this == other2) return true;
+ return false;
+}
+
+std::pair<Tree, Input> Input::fetch(ref<Store> store) const
+{
+ if (!scheme)
+ throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
+
+ /* The tree may already be in the Nix store, or it could be
+ substituted (which is often faster than fetching from the
+ original source). So check that. */
+ if (hasAllInfo()) {
+ try {
+ auto storePath = computeStorePath(*store);
+
+ store->ensurePath(storePath);
+
+ debug("using substituted/cached input '%s' in '%s'",
+ to_string(), store->printStorePath(storePath));
+
+ auto actualPath = store->toRealPath(storePath);
+
+ return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
+ } catch (Error & e) {
+ debug("substitution of input '%s' failed: %s", to_string(), e.what());
+ }
+ }
+
+ auto [tree, input] = scheme->fetch(store, *this);
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
- if (!tree.info.narHash)
- tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+ auto narHash = store->queryPathInfo(tree.storePath)->narHash;
+ input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
+
+ if (auto prevNarHash = getNarHash()) {
+ if (narHash != *prevNarHash)
+ throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
+ }
- if (input->narHash)
- assert(input->narHash == tree.info.narHash);
+ if (auto prevLastModified = getLastModified()) {
+ if (input.getLastModified() != prevLastModified)
+ throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
+ input.to_string(), *prevLastModified);
+ }
- if (narHash && narHash != input->narHash)
- throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
+ if (auto prevRevCount = getRevCount()) {
+ if (input.getRevCount() != prevRevCount)
+ throw Error("'revCount' attribute mismatch in input '%s', expected %d",
+ input.to_string(), *prevRevCount);
+ }
+
+ input.immutable = true;
+
+ assert(input.hasAllInfo());
return {std::move(tree), input};
}
+Input Input::applyOverrides(
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) const
+{
+ if (!scheme) return *this;
+ return scheme->applyOverrides(*this, ref, rev);
+}
+
+void Input::clone(const Path & destDir) const
+{
+ assert(scheme);
+ scheme->clone(*this, destDir);
+}
+
+std::optional<Path> Input::getSourcePath() const
+{
+ assert(scheme);
+ return scheme->getSourcePath(*this);
+}
+
+void Input::markChangedFile(
+ std::string_view file,
+ std::optional<std::string> commitMsg) const
+{
+ assert(scheme);
+ return scheme->markChangedFile(*this, file, commitMsg);
+}
+
+StorePath Input::computeStorePath(Store & store) const
+{
+ auto narHash = getNarHash();
+ if (!narHash)
+ throw Error("cannot compute store path for mutable input '%s'", to_string());
+ return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+}
+
+std::string Input::getType() const
+{
+ return getStrAttr(attrs, "type");
+}
+
+std::optional<Hash> Input::getNarHash() const
+{
+ if (auto s = maybeGetStrAttr(attrs, "narHash"))
+ // FIXME: require SRI hash.
+ return newHashAllowEmpty(*s, htSHA256);
+ return {};
+}
+
+std::optional<std::string> Input::getRef() const
+{
+ if (auto s = maybeGetStrAttr(attrs, "ref"))
+ return *s;
+ return {};
+}
+
+std::optional<Hash> Input::getRev() const
+{
+ if (auto s = maybeGetStrAttr(attrs, "rev"))
+ return Hash(*s, htSHA1);
+ return {};
+}
+
+std::optional<uint64_t> Input::getRevCount() const
+{
+ if (auto n = maybeGetIntAttr(attrs, "revCount"))
+ return *n;
+ return {};
+}
+
+std::optional<time_t> Input::getLastModified() const
+{
+ if (auto n = maybeGetIntAttr(attrs, "lastModified"))
+ return *n;
+ return {};
+}
+
+ParsedURL InputScheme::toURL(const Input & input)
+{
+ throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
+}
+
+Input InputScheme::applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev)
+{
+ if (ref)
+ throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
+ if (rev)
+ throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
+ return input;
+}
+
+std::optional<Path> InputScheme::getSourcePath(const Input & input)
+{
+ return {};
+}
+
+void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
+{
+ assert(false);
+}
+
+void InputScheme::clone(const Input & input, const Path & destDir)
+{
+ throw Error("do not know how to clone input '%s'", input.to_string());
+}
+
}
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 59a58ae67..be71b786b 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -3,7 +3,6 @@
#include "types.hh"
#include "hash.hh"
#include "path.hh"
-#include "tree-info.hh"
#include "attrs.hh"
#include "url.hh"
@@ -13,73 +12,101 @@ namespace nix { class Store; }
namespace nix::fetchers {
-struct Input;
-
struct Tree
{
Path actualPath;
StorePath storePath;
- TreeInfo info;
+ Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
};
-struct Input : std::enable_shared_from_this<Input>
+struct InputScheme;
+
+struct Input
{
- std::optional<Hash> narHash; // FIXME: implement
+ friend class InputScheme;
+
+ std::shared_ptr<InputScheme> scheme; // note: can be null
+ Attrs attrs;
+ bool immutable = false;
+ bool direct = true;
+
+public:
+ static Input fromURL(const std::string & url);
- virtual std::string type() const = 0;
+ static Input fromURL(const ParsedURL & url);
- virtual ~Input() { }
+ static Input fromAttrs(Attrs && attrs);
- virtual bool operator ==(const Input & other) const { return false; }
+ ParsedURL toURL() const;
+
+ std::string to_string() const;
+
+ Attrs toAttrs() const;
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
- virtual bool isDirect() const { return true; }
+ bool isDirect() const { return direct; }
/* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
- virtual bool isImmutable() const { return (bool) narHash; }
+ bool isImmutable() const { return immutable; }
- virtual bool contains(const Input & other) const { return false; }
+ bool hasAllInfo() const;
- virtual std::optional<std::string> getRef() const { return {}; }
+ bool operator ==(const Input & other) const;
- virtual std::optional<Hash> getRev() const { return {}; }
+ bool contains(const Input & other) const;
- virtual ParsedURL toURL() const = 0;
+ std::pair<Tree, Input> fetch(ref<Store> store) const;
- std::string to_string() const
- {
- return toURL().to_string();
- }
+ Input applyOverrides(
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) const;
- Attrs toAttrs() const;
+ void clone(const Path & destDir) const;
- std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+ std::optional<Path> getSourcePath() const;
-private:
+ void markChangedFile(
+ std::string_view file,
+ std::optional<std::string> commitMsg) const;
- virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+ StorePath computeStorePath(Store & store) const;
- virtual Attrs toAttrsInternal() const = 0;
+ // Convience functions for common attributes.
+ std::string getType() const;
+ std::optional<Hash> getNarHash() const;
+ std::optional<std::string> getRef() const;
+ std::optional<Hash> getRev() const;
+ std::optional<uint64_t> getRevCount() const;
+ std::optional<time_t> getLastModified() const;
};
struct InputScheme
{
- virtual ~InputScheme() { }
+ virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
- virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+ virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
- virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
-};
+ virtual ParsedURL toURL(const Input & input);
+
+ virtual bool hasAllInfo(const Input & input) = 0;
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
+ virtual Input applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev);
-std::unique_ptr<Input> inputFromURL(const std::string & url);
+ virtual void clone(const Input & input, const Path & destDir);
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
+ virtual std::optional<Path> getSourcePath(const Input & input);
+
+ virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
+
+ virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
+};
-void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
@@ -94,7 +121,7 @@ DownloadFileResult downloadFile(
const std::string & name,
bool immutable);
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 75ce5ee8b..5d38e0c2b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -22,80 +22,152 @@ static bool isNotDotGitDirectory(const Path & path)
return not std::regex_match(path, gitDirRegex);
}
-struct GitInput : Input
+struct GitInputScheme : InputScheme
{
- ParsedURL url;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
- bool shallow = false;
- bool submodules = false;
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "git" &&
+ url.scheme != "git+http" &&
+ url.scheme != "git+https" &&
+ url.scheme != "git+ssh" &&
+ url.scheme != "git+file") return {};
+
+ auto url2(url);
+ if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+ url2.query.clear();
+
+ Attrs attrs;
+ attrs.emplace("type", "git");
+
+ for (auto &[name, value] : url.query) {
+ if (name == "rev" || name == "ref")
+ attrs.emplace(name, value);
+ else if (name == "shallow")
+ attrs.emplace(name, Explicit<bool> { value == "1" });
+ else
+ url2.query.emplace(name, value);
+ }
+
+ attrs.emplace("url", url2.to_string());
+
+ return inputFromAttrs(attrs);
+ }
- GitInput(const ParsedURL & url) : url(url)
- { }
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
+ {
+ if (maybeGetStrAttr(attrs, "type") != "git") return {};
- std::string type() const override { return "git"; }
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+ throw Error("unsupported Git input attribute '%s'", name);
+
+ parseURL(getStrAttr(attrs, "url"));
+ maybeGetBoolAttr(attrs, "shallow");
+ maybeGetBoolAttr(attrs, "submodules");
+
+ if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+ if (std::regex_search(*ref, badGitRefRegex))
+ throw BadURL("invalid Git branch/tag name '%s'", *ref);
+ }
+
+ Input input;
+ input.attrs = attrs;
+ return input;
+ }
- bool operator ==(const Input & other) const override
+ ParsedURL toURL(const Input & input) override
{
- auto other2 = dynamic_cast<const GitInput *>(&other);
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ if (url.scheme != "git") url.scheme = "git+" + url.scheme;
+ if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+ if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
+ if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+ url.query.insert_or_assign("shallow", "1");
+ return url;
+ }
+
+ bool hasAllInfo(const Input & input) override
+ {
+ bool maybeDirty = !input.getRef();
+ bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
- other2
- && url == other2->url
- && rev == other2->rev
- && ref == other2->ref;
+ maybeGetIntAttr(input.attrs, "lastModified")
+ && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
- bool isImmutable() const override
+ Input applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
{
- return (bool) rev || narHash;
+ auto res(input);
+ if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) res.attrs.insert_or_assign("ref", *ref);
+ if (!res.getRef() && res.getRev())
+ throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
+ return res;
}
- std::optional<std::string> getRef() const override { return ref; }
+ void clone(const Input & input, const Path & destDir) override
+ {
+ auto [isLocal, actualUrl] = getActualUrl(input);
+
+ Strings args = {"clone"};
+
+ args.push_back(actualUrl);
- std::optional<Hash> getRev() const override { return rev; }
+ if (auto ref = input.getRef()) {
+ args.push_back("--branch");
+ args.push_back(*ref);
+ }
+
+ if (input.getRev()) throw Error("cloning a specific revision is not implemented");
+
+ args.push_back(destDir);
- ParsedURL toURL() const override
+ runProgram("git", true, args);
+ }
+
+ std::optional<Path> getSourcePath(const Input & input) override
{
- ParsedURL url2(url);
- if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
- if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
- if (ref) url2.query.insert_or_assign("ref", *ref);
- if (shallow) url2.query.insert_or_assign("shallow", "1");
- return url2;
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ if (url.scheme == "file" && !input.getRef() && !input.getRev())
+ return url.path;
+ return {};
}
- Attrs toAttrsInternal() const override
+ void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
- Attrs attrs;
- attrs.emplace("url", url.to_string());
- if (ref)
- attrs.emplace("ref", *ref);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- if (shallow)
- attrs.emplace("shallow", true);
- if (submodules)
- attrs.emplace("submodules", true);
- return attrs;
+ auto sourcePath = getSourcePath(input);
+ assert(sourcePath);
+
+ runProgram("git", true,
+ { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) });
+
+ if (commitMsg)
+ runProgram("git", true,
+ { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
}
- std::pair<bool, std::string> getActualUrl() const
+ std::pair<bool, std::string> getActualUrl(const Input & input) const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base};
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
- auto input = std::make_shared<GitInput>(*this);
+ Input input(_input);
- assert(!rev || rev->type == htSHA1);
+ bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+ bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
@@ -106,39 +178,35 @@ struct GitInput : Input
return Attrs({
{"type", cacheType},
{"name", name},
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
- -> std::pair<Tree, std::shared_ptr<const Input>>
+ -> std::pair<Tree, Input>
{
- assert(input->rev);
- assert(!rev || rev == input->rev);
+ assert(input.getRev());
+ assert(!_input.getRev() || _input.getRev() == input.getRev());
+ if (!shallow)
+ input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+ input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {
- Tree {
- .actualPath = store->toRealPath(storePath),
- .storePath = std::move(storePath),
- .info = TreeInfo {
- .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
- .lastModified = getIntAttr(infoAttrs, "lastModified"),
- },
- },
+ Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
};
- if (rev) {
+ if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
- auto [isLocal, actualUrl_] = getActualUrl();
+ auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
- if (!input->ref && !input->rev && isLocal) {
+ if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false;
/* Check whether this repo has any commits. There are
@@ -197,35 +265,35 @@ struct GitInput : Input
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
- auto tree = Tree {
- .actualPath = store->printStorePath(storePath),
- .storePath = std::move(storePath),
- .info = TreeInfo {
- // FIXME: maybe we should use the timestamp of the last
- // modified dirty file?
- .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
- }
- };
+ // FIXME: maybe we should use the timestamp of the last
+ // modified dirty file?
+ input.attrs.insert_or_assign(
+ "lastModified",
+ haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
- return {std::move(tree), input};
+ return {
+ Tree(store->printStorePath(storePath), std::move(storePath)),
+ input
+ };
}
}
- if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+ if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
- {"ref", *input->ref},
+ {"ref", *input.getRef()},
});
Path repoDir;
if (isLocal) {
- if (!input->rev)
- input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+ if (!input.getRev())
+ input.attrs.insert_or_assign("rev",
+ Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
@@ -233,8 +301,8 @@ struct GitInput : Input
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
- if (!rev || rev == rev2) {
- input->rev = rev2;
+ if (!input.getRev() || input.getRev() == rev2) {
+ input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@@ -248,18 +316,18 @@ struct GitInput : Input
}
Path localRefFile =
- input->ref->compare(0, 5, "refs/") == 0
- ? cacheDir + "/" + *input->ref
- : cacheDir + "/refs/heads/" + *input->ref;
+ input.getRef()->compare(0, 5, "refs/") == 0
+ ? cacheDir + "/" + *input.getRef()
+ : cacheDir + "/refs/heads/" + *input.getRef();
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
- if (input->rev) {
+ if (input.getRev()) {
try {
- runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+ runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
@@ -282,9 +350,10 @@ struct GitInput : Input
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
- auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
- ? *input->ref
- : "refs/heads/" + *input->ref;
+ auto ref = input.getRef();
+ auto fetchRef = ref->compare(0, 5, "refs/") == 0
+ ? *ref
+ : "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
@@ -300,8 +369,8 @@ struct GitInput : Input
utimes(localRefFile.c_str(), times);
}
- if (!input->rev)
- input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+ if (!input.getRev())
+ input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -311,7 +380,7 @@ struct GitInput : Input
// FIXME: check whether rev is an ancestor of ref.
- printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+ printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
@@ -333,7 +402,7 @@ struct GitInput : Input
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
- runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
+ runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
@@ -342,7 +411,7 @@ struct GitInput : Input
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
- RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+ RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
@@ -352,18 +421,18 @@ struct GitInput : Input
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
- auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+ auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
Attrs infoAttrs({
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
{"lastModified", lastModified},
});
if (!shallow)
infoAttrs.insert_or_assign("revCount",
- std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+ std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
- if (!this->rev)
+ if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@@ -382,60 +451,6 @@ struct GitInput : Input
}
};
-struct GitInputScheme : InputScheme
-{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
- {
- if (url.scheme != "git" &&
- url.scheme != "git+http" &&
- url.scheme != "git+https" &&
- url.scheme != "git+ssh" &&
- url.scheme != "git+file") return nullptr;
-
- auto url2(url);
- if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
- url2.query.clear();
-
- Attrs attrs;
- attrs.emplace("type", "git");
-
- for (auto &[name, value] : url.query) {
- if (name == "rev" || name == "ref")
- attrs.emplace(name, value);
- else
- url2.query.emplace(name, value);
- }
-
- attrs.emplace("url", url2.to_string());
-
- return inputFromAttrs(attrs);
- }
-
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
- {
- if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
- for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
- throw Error("unsupported Git input attribute '%s'", name);
-
- auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
- if (auto ref = maybeGetStrAttr(attrs, "ref")) {
- if (std::regex_search(*ref, badGitRefRegex))
- throw BadURL("invalid Git branch/tag name '%s'", *ref);
- input->ref = *ref;
- }
- if (auto rev = maybeGetStrAttr(attrs, "rev"))
- input->rev = Hash(*rev, htSHA1);
-
- input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
-
- input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
-
- return input;
- }
-};
-
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 0bee1d6b3..8bb7c2c1d 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -8,81 +8,142 @@
namespace nix::fetchers {
-std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+// A github or gitlab url
+const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
-struct GitHubInput : Input
+struct GitArchiveInputScheme : InputScheme
{
- std::string owner;
- std::string repo;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
+ virtual std::string type() = 0;
- std::string type() const override { return "github"; }
-
- bool operator ==(const Input & other) const override
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
{
- auto other2 = dynamic_cast<const GitHubInput *>(&other);
- return
- other2
- && owner == other2->owner
- && repo == other2->repo
- && rev == other2->rev
- && ref == other2->ref;
+ if (url.scheme != type()) return {};
+
+ auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+ std::optional<Hash> rev;
+ std::optional<std::string> ref;
+ std::optional<std::string> host_url;
+
+ if (path.size() == 2) {
+ } else if (path.size() == 3) {
+ if (std::regex_match(path[2], revRegex))
+ rev = Hash(path[2], htSHA1);
+ else if (std::regex_match(path[2], refRegex))
+ ref = path[2];
+ else
+ throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+ } else
+ throw BadURL("URL '%s' is invalid", url.url);
+
+ for (auto &[name, value] : url.query) {
+ if (name == "rev") {
+ if (rev)
+ throw BadURL("URL '%s' contains multiple commit hashes", url.url);
+ rev = Hash(value, htSHA1);
+ }
+ else if (name == "ref") {
+ if (!std::regex_match(value, refRegex))
+ throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
+ if (ref)
+ throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
+ ref = value;
+ }
+ else if (name == "url") {
+ if (!std::regex_match(value, urlRegex))
+ throw BadURL("URL '%s' contains an invalid instance url", url.url);
+ host_url = value;
+ }
+ // FIXME: barf on unsupported attributes
+ }
+
+ if (ref && rev)
+ throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
+
+ Input input;
+ input.attrs.insert_or_assign("type", type());
+ input.attrs.insert_or_assign("owner", path[0]);
+ input.attrs.insert_or_assign("repo", path[1]);
+ if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) input.attrs.insert_or_assign("ref", *ref);
+ if (host_url) input.attrs.insert_or_assign("url", *host_url);
+
+ return input;
}
- bool isImmutable() const override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
- return (bool) rev || narHash;
- }
+ if (maybeGetStrAttr(attrs, "type") != type()) return {};
+
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+ throw Error("unsupported input attribute '%s'", name);
- std::optional<std::string> getRef() const override { return ref; }
+ getStrAttr(attrs, "owner");
+ getStrAttr(attrs, "repo");
- std::optional<Hash> getRev() const override { return rev; }
+ Input input;
+ input.attrs = attrs;
+ return input;
+ }
- ParsedURL toURL() const override
+ ParsedURL toURL(const Input & input) override
{
+ auto owner = getStrAttr(input.attrs, "owner");
+ auto repo = getStrAttr(input.attrs, "repo");
+ auto ref = input.getRef();
+ auto rev = input.getRev();
auto path = owner + "/" + repo;
assert(!(ref && rev));
if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false);
return ParsedURL {
- .scheme = "github",
+ .scheme = type(),
.path = path,
};
}
- Attrs toAttrsInternal() const override
+ bool hasAllInfo(const Input & input) override
{
- Attrs attrs;
- attrs.emplace("owner", owner);
- attrs.emplace("repo", repo);
- if (ref)
- attrs.emplace("ref", *ref);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- return attrs;
+ return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ Input applyOverrides(
+ const Input & _input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
{
- auto rev = this->rev;
- auto ref = this->ref.value_or("master");
-
- if (!rev) {
- auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
- owner, repo, ref);
- auto json = nlohmann::json::parse(
- readFile(
- store->toRealPath(
- downloadFile(store, url, "source", false).storePath)));
- rev = Hash(std::string { json["sha"] }, htSHA1);
- debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+ auto input(_input);
+ if (rev && ref)
+ throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'",
+ rev->gitRev(), *ref, input.to_string());
+ if (rev) {
+ input.attrs.insert_or_assign("rev", rev->gitRev());
+ input.attrs.erase("ref");
}
+ if (ref) {
+ input.attrs.insert_or_assign("ref", *ref);
+ input.attrs.erase("rev");
+ }
+ return input;
+ }
+
+ virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
+
+ virtual std::string getDownloadUrl(const Input & input) const = 0;
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+ {
+ Input input(_input);
- auto input = std::make_shared<GitHubInput>(*this);
- input->ref = {};
- input->rev = *rev;
+ if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
+
+ auto rev = input.getRev();
+ if (!rev) rev = getRevFromRef(store, input);
+
+ input.attrs.erase("ref");
+ input.attrs.insert_or_assign("rev", rev->gitRev());
Attrs immutableAttrs({
{"type", "git-tarball"},
@@ -90,36 +151,25 @@ struct GitHubInput : Input
});
if (auto res = getCache()->lookup(store, immutableAttrs)) {
+ input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return {
- Tree{
- .actualPath = store->toRealPath(res->second),
- .storePath = std::move(res->second),
- .info = TreeInfo {
- .lastModified = getIntAttr(res->first, "lastModified"),
- },
- },
+ Tree(store->toRealPath(res->second), std::move(res->second)),
input
};
}
- // FIXME: use regular /archive URLs instead? api.github.com
- // might have stricter rate limits.
+ auto url = getDownloadUrl(input);
- auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
- owner, repo, rev->to_string(Base16, false));
+ auto [tree, lastModified] = downloadTarball(store, url, "source", true);
- std::string accessToken = settings.githubAccessToken.get();
- if (accessToken != "")
- url += "?access_token=" + accessToken;
-
- auto tree = downloadTarball(store, url, "source", true);
+ input.attrs.insert_or_assign("lastModified", lastModified);
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
- {"lastModified", *tree.info.lastModified}
+ {"lastModified", lastModified}
},
tree.storePath,
true);
@@ -128,68 +178,96 @@ struct GitHubInput : Input
}
};
-struct GitHubInputScheme : InputScheme
+struct GitHubInputScheme : GitArchiveInputScheme
{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+ std::string type() override { return "github"; }
+
+ Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
- if (url.scheme != "github") return nullptr;
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ auto json = nlohmann::json::parse(
+ readFile(
+ store->toRealPath(
+ downloadFile(store, url, "source", false).storePath)));
+ auto rev = Hash(std::string { json["sha"] }, htSHA1);
+ debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+ return rev;
+ }
- auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
- auto input = std::make_unique<GitHubInput>();
+ std::string getDownloadUrl(const Input & input) const override
+ {
+ // FIXME: use regular /archive URLs instead? api.github.com
+ // might have stricter rate limits.
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ input.getRev()->to_string(Base16, false));
- if (path.size() == 2) {
- } else if (path.size() == 3) {
- if (std::regex_match(path[2], revRegex))
- input->rev = Hash(path[2], htSHA1);
- else if (std::regex_match(path[2], refRegex))
- input->ref = path[2];
- else
- throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
- } else
- throw BadURL("GitHub URL '%s' is invalid", url.url);
+ std::string accessToken = settings.githubAccessToken.get();
+ if (accessToken != "")
+ url += "?access_token=" + accessToken;
- for (auto &[name, value] : url.query) {
- if (name == "rev") {
- if (input->rev)
- throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
- input->rev = Hash(value, htSHA1);
- }
- else if (name == "ref") {
- if (!std::regex_match(value, refRegex))
- throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
- if (input->ref)
- throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
- input->ref = value;
- }
- }
+ return url;
+ }
- if (input->ref && input->rev)
- throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
+ void clone(const Input & input, const Path & destDir) override
+ {
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .clone(destDir);
+ }
+};
- input->owner = path[0];
- input->repo = path[1];
+struct GitLabInputScheme : GitArchiveInputScheme
+{
+ std::string type() override { return "gitlab"; }
- return input;
+ Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
+ {
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ auto json = nlohmann::json::parse(
+ readFile(
+ store->toRealPath(
+ downloadFile(store, url, "source", false).storePath)));
+ auto rev = Hash(std::string(json[0]["id"]), htSHA1);
+ debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+ return rev;
}
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+ std::string getDownloadUrl(const Input & input) const override
{
- if (maybeGetStrAttr(attrs, "type") != "github") return {};
+ // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ input.getRev()->to_string(Base16, false));
- for (auto & [name, value] : attrs)
- if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
- throw Error("unsupported GitHub input attribute '%s'", name);
-
- auto input = std::make_unique<GitHubInput>();
- input->owner = getStrAttr(attrs, "owner");
- input->repo = getStrAttr(attrs, "repo");
- input->ref = maybeGetStrAttr(attrs, "ref");
- if (auto rev = maybeGetStrAttr(attrs, "rev"))
- input->rev = Hash(*rev, htSHA1);
- return input;
+ /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
+ std::string accessToken = settings.githubAccessToken.get();
+ if (accessToken != "")
+ url += "?access_token=" + accessToken;*/
+
+ return url;
+ }
+
+ void clone(const Input & input, const Path & destDir) override
+ {
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ // FIXME: get username somewhere
+ Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .clone(destDir);
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
}
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
new file mode 100644
index 000000000..91dc83740
--- /dev/null
+++ b/src/libfetchers/indirect.cc
@@ -0,0 +1,104 @@
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct IndirectInputScheme : InputScheme
+{
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "flake") return {};
+
+ auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+ std::optional<Hash> rev;
+ std::optional<std::string> ref;
+
+ if (path.size() == 1) {
+ } else if (path.size() == 2) {
+ if (std::regex_match(path[1], revRegex))
+ rev = Hash(path[1], htSHA1);
+ else if (std::regex_match(path[1], refRegex))
+ ref = path[1];
+ else
+ throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
+ } else if (path.size() == 3) {
+ if (!std::regex_match(path[1], refRegex))
+ throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
+ ref = path[1];
+ if (!std::regex_match(path[2], revRegex))
+ throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
+ rev = Hash(path[2], htSHA1);
+ } else
+ throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+ std::string id = path[0];
+ if (!std::regex_match(id, flakeRegex))
+ throw BadURL("'%s' is not a valid flake ID", id);
+
+ // FIXME: forbid query params?
+
+ Input input;
+ input.direct = false;
+ input.attrs.insert_or_assign("type", "indirect");
+ input.attrs.insert_or_assign("id", id);
+ if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) input.attrs.insert_or_assign("ref", *ref);
+
+ return input;
+ }
+
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
+ {
+ if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
+
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
+ throw Error("unsupported indirect input attribute '%s'", name);
+
+ auto id = getStrAttr(attrs, "id");
+ if (!std::regex_match(id, flakeRegex))
+ throw BadURL("'%s' is not a valid flake ID", id);
+
+ Input input;
+ input.direct = false;
+ input.attrs = attrs;
+ return input;
+ }
+
+ ParsedURL toURL(const Input & input) override
+ {
+ ParsedURL url;
+ url.scheme = "flake";
+ url.path = getStrAttr(input.attrs, "id");
+ if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
+ if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
+ return url;
+ }
+
+ bool hasAllInfo(const Input & input) override
+ {
+ return false;
+ }
+
+ Input applyOverrides(
+ const Input & _input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
+ {
+ auto input(_input);
+ if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) input.attrs.insert_or_assign("ref", *ref);
+ return input;
+ }
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ {
+ throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
+ }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+
+}
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 2e0d4bf4d..c48cb6fd1 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -10,76 +10,124 @@ using namespace std::string_literals;
namespace nix::fetchers {
-struct MercurialInput : Input
+struct MercurialInputScheme : InputScheme
{
- ParsedURL url;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "hg+http" &&
+ url.scheme != "hg+https" &&
+ url.scheme != "hg+ssh" &&
+ url.scheme != "hg+file") return {};
- MercurialInput(const ParsedURL & url) : url(url)
- { }
+ auto url2(url);
+ url2.scheme = std::string(url2.scheme, 3);
+ url2.query.clear();
+
+ Attrs attrs;
+ attrs.emplace("type", "hg");
- std::string type() const override { return "hg"; }
+ for (auto &[name, value] : url.query) {
+ if (name == "rev" || name == "ref")
+ attrs.emplace(name, value);
+ else
+ url2.query.emplace(name, value);
+ }
+
+ attrs.emplace("url", url2.to_string());
+
+ return inputFromAttrs(attrs);
+ }
- bool operator ==(const Input & other) const override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
- auto other2 = dynamic_cast<const MercurialInput *>(&other);
- return
- other2
- && url == other2->url
- && rev == other2->rev
- && ref == other2->ref;
+ if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+ throw Error("unsupported Mercurial input attribute '%s'", name);
+
+ parseURL(getStrAttr(attrs, "url"));
+
+ if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+ if (!std::regex_match(*ref, refRegex))
+ throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+ }
+
+ Input input;
+ input.attrs = attrs;
+ return input;
}
- bool isImmutable() const override
+ ParsedURL toURL(const Input & input) override
{
- return (bool) rev || narHash;
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ url.scheme = "hg+" + url.scheme;
+ if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+ if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
+ return url;
}
- std::optional<std::string> getRef() const override { return ref; }
+ bool hasAllInfo(const Input & input) override
+ {
+ // FIXME: ugly, need to distinguish between dirty and clean
+ // default trees.
+ return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
+ }
- std::optional<Hash> getRev() const override { return rev; }
+ Input applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
+ {
+ auto res(input);
+ if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) res.attrs.insert_or_assign("ref", *ref);
+ return res;
+ }
- ParsedURL toURL() const override
+ std::optional<Path> getSourcePath(const Input & input) override
{
- ParsedURL url2(url);
- url2.scheme = "hg+" + url2.scheme;
- if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
- if (ref) url2.query.insert_or_assign("ref", *ref);
- return url;
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ if (url.scheme == "file" && !input.getRef() && !input.getRev())
+ return url.path;
+ return {};
}
- Attrs toAttrsInternal() const override
+ void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
- Attrs attrs;
- attrs.emplace("url", url.to_string());
- if (ref)
- attrs.emplace("ref", *ref);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- return attrs;
+ auto sourcePath = getSourcePath(input);
+ assert(sourcePath);
+
+ // FIXME: shut up if file is already tracked.
+ runProgram("hg", true,
+ { "add", *sourcePath + "/" + std::string(file) });
+
+ if (commitMsg)
+ runProgram("hg", true,
+ { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
- std::pair<bool, std::string> getActualUrl() const
+ std::pair<bool, std::string> getActualUrl(const Input & input) const
{
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base};
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
- auto input = std::make_shared<MercurialInput>(*this);
+ Input input(_input);
- auto [isLocal, actualUrl_] = getActualUrl();
+ auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// FIXME: return lastModified.
// FIXME: don't clone local repositories.
- if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+ if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
@@ -94,7 +142,7 @@ struct MercurialInput : Input
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
- input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+ input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
@@ -116,60 +164,54 @@ struct MercurialInput : Input
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
- return {Tree {
- .actualPath = store->printStorePath(storePath),
- .storePath = std::move(storePath),
- }, input};
+ return {
+ Tree(store->printStorePath(storePath), std::move(storePath)),
+ input
+ };
}
}
- if (!input->ref) input->ref = "default";
+ if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", "hg"},
{"name", name},
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
- -> std::pair<Tree, std::shared_ptr<const Input>>
+ -> std::pair<Tree, Input>
{
- assert(input->rev);
- assert(!rev || rev == input->rev);
+ assert(input.getRev());
+ assert(!_input.getRev() || _input.getRev() == input.getRev());
+ input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
return {
- Tree{
- .actualPath = store->toRealPath(storePath),
- .storePath = std::move(storePath),
- .info = TreeInfo {
- .revCount = getIntAttr(infoAttrs, "revCount"),
- },
- },
+ Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
};
- if (input->rev) {
+ if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
- assert(input->rev || input->ref);
- auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+ auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
Attrs mutableAttrs({
{"type", "hg"},
{"name", name},
{"url", actualUrl},
- {"ref", *input->ref},
+ {"ref", *input.getRef()},
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
- if (!rev || rev == rev2) {
- input->rev = rev2;
+ if (!input.getRev() || input.getRev() == rev2) {
+ input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@@ -178,10 +220,10 @@ struct MercurialInput : Input
/* If this is a commit hash that we already have, we don't
have to pull again. */
- if (!(input->rev
+ if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
- RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+ RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
@@ -210,9 +252,9 @@ struct MercurialInput : Input
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
- input->rev = Hash(tokens[0], htSHA1);
+ input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
- input->ref = tokens[2];
+ input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
@@ -220,18 +262,18 @@ struct MercurialInput : Input
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
- runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+ runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir);
Attrs infoAttrs({
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
{"revCount", (int64_t) revCount},
});
- if (!this->rev)
+ if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@@ -250,54 +292,6 @@ struct MercurialInput : Input
}
};
-struct MercurialInputScheme : InputScheme
-{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
- {
- if (url.scheme != "hg+http" &&
- url.scheme != "hg+https" &&
- url.scheme != "hg+ssh" &&
- url.scheme != "hg+file") return nullptr;
-
- auto url2(url);
- url2.scheme = std::string(url2.scheme, 3);
- url2.query.clear();
-
- Attrs attrs;
- attrs.emplace("type", "hg");
-
- for (auto &[name, value] : url.query) {
- if (name == "rev" || name == "ref")
- attrs.emplace(name, value);
- else
- url2.query.emplace(name, value);
- }
-
- attrs.emplace("url", url2.to_string());
-
- return inputFromAttrs(attrs);
- }
-
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
- {
- if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
- for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev")
- throw Error("unsupported Mercurial input attribute '%s'", name);
-
- auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
- if (auto ref = maybeGetStrAttr(attrs, "ref")) {
- if (!std::regex_match(*ref, refRegex))
- throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
- input->ref = *ref;
- }
- if (auto rev = maybeGetStrAttr(attrs, "rev"))
- input->rev = Hash(*rev, htSHA1);
- return input;
- }
-};
-
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index ba2cc192e..99d4b4e8f 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -3,65 +3,86 @@
namespace nix::fetchers {
-struct PathInput : Input
+struct PathInputScheme : InputScheme
{
- Path path;
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "path") return {};
- /* Allow the user to pass in "fake" tree info attributes. This is
- useful for making a pinned tree work the same as the repository
- from which is exported
- (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
- std::optional<Hash> rev;
- std::optional<uint64_t> revCount;
- std::optional<time_t> lastModified;
+ if (url.authority && *url.authority != "")
+ throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
- std::string type() const override { return "path"; }
+ Input input;
+ input.attrs.insert_or_assign("type", "path");
+ input.attrs.insert_or_assign("path", url.path);
- std::optional<Hash> getRev() const override { return rev; }
+ for (auto & [name, value] : url.query)
+ if (name == "rev" || name == "narHash")
+ input.attrs.insert_or_assign(name, value);
+ else if (name == "revCount" || name == "lastModified") {
+ uint64_t n;
+ if (!string2Int(value, n))
+ throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+ input.attrs.insert_or_assign(name, n);
+ }
+ else
+ throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
- bool operator ==(const Input & other) const override
- {
- auto other2 = dynamic_cast<const PathInput *>(&other);
- return
- other2
- && path == other2->path
- && rev == other2->rev
- && revCount == other2->revCount
- && lastModified == other2->lastModified;
+ return input;
}
- bool isImmutable() const override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
- return (bool) narHash;
+ if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
+ getStrAttr(attrs, "path");
+
+ for (auto & [name, value] : attrs)
+ /* Allow the user to pass in "fake" tree info
+ attributes. This is useful for making a pinned tree
+ work the same as the repository from which is exported
+ (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+ if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
+ // checked in Input::fromAttrs
+ ;
+ else
+ throw Error("unsupported path input attribute '%s'", name);
+
+ Input input;
+ input.attrs = attrs;
+ return input;
}
- ParsedURL toURL() const override
+ ParsedURL toURL(const Input & input) override
{
- auto query = attrsToQuery(toAttrsInternal());
+ auto query = attrsToQuery(input.attrs);
query.erase("path");
+ query.erase("type");
return ParsedURL {
.scheme = "path",
- .path = path,
+ .path = getStrAttr(input.attrs, "path"),
.query = query,
};
}
- Attrs toAttrsInternal() const override
+ bool hasAllInfo(const Input & input) override
+ {
+ return true;
+ }
+
+ std::optional<Path> getSourcePath(const Input & input) override
{
- Attrs attrs;
- attrs.emplace("path", path);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- if (revCount)
- attrs.emplace("revCount", *revCount);
- if (lastModified)
- attrs.emplace("lastModified", *lastModified);
- return attrs;
+ return getStrAttr(input.attrs, "path");
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
- auto input = std::make_shared<PathInput>(*this);
+ // nothing to do
+ }
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ {
+ auto path = getStrAttr(input.attrs, "path");
// FIXME: check whether access to 'path' is allowed.
@@ -74,72 +95,10 @@ struct PathInput : Input
// FIXME: try to substitute storePath.
storePath = store->addToStore("source", path);
- return
- {
- Tree {
- .actualPath = store->toRealPath(*storePath),
- .storePath = std::move(*storePath),
- .info = TreeInfo {
- .revCount = revCount,
- .lastModified = lastModified
- }
- },
- input
- };
- }
-
-};
-
-struct PathInputScheme : InputScheme
-{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
- {
- if (url.scheme != "path") return nullptr;
-
- auto input = std::make_unique<PathInput>();
- input->path = url.path;
-
- for (auto & [name, value] : url.query)
- if (name == "rev")
- input->rev = Hash(value, htSHA1);
- else if (name == "revCount") {
- uint64_t revCount;
- if (!string2Int(value, revCount))
- throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
- input->revCount = revCount;
- }
- else if (name == "lastModified") {
- time_t lastModified;
- if (!string2Int(value, lastModified))
- throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
- input->lastModified = lastModified;
- }
- else
- throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
-
- return input;
- }
-
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
- {
- if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
- auto input = std::make_unique<PathInput>();
- input->path = getStrAttr(attrs, "path");
-
- for (auto & [name, value] : attrs)
- if (name == "rev")
- input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
- else if (name == "revCount")
- input->revCount = getIntAttr(attrs, "revCount");
- else if (name == "lastModified")
- input->lastModified = getIntAttr(attrs, "lastModified");
- else if (name == "type" || name == "path")
- ;
- else
- throw Error("unsupported path input attribute '%s'", name);
-
- return input;
+ return {
+ Tree(store->toRealPath(*storePath), std::move(*storePath)),
+ input
+ };
}
};
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
new file mode 100644
index 000000000..d4134ce29
--- /dev/null
+++ b/src/libfetchers/registry.cc
@@ -0,0 +1,212 @@
+#include "registry.hh"
+#include "fetchers.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::shared_ptr<Registry> Registry::read(
+ const Path & path, RegistryType type)
+{
+ auto registry = std::make_shared<Registry>(type);
+
+ if (!pathExists(path))
+ return std::make_shared<Registry>(type);
+
+ try {
+
+ auto json = nlohmann::json::parse(readFile(path));
+
+ auto version = json.value("version", 0);
+
+ if (version == 2) {
+ for (auto & i : json["flakes"]) {
+ auto toAttrs = jsonToAttrs(i["to"]);
+ Attrs extraAttrs;
+ auto j = toAttrs.find("dir");
+ if (j != toAttrs.end()) {
+ extraAttrs.insert(*j);
+ toAttrs.erase(j);
+ }
+ auto exact = i.find("exact");
+ registry->entries.push_back(
+ Entry {
+ .from = Input::fromAttrs(jsonToAttrs(i["from"])),
+ .to = Input::fromAttrs(std::move(toAttrs)),
+ .extraAttrs = extraAttrs,
+ .exact = exact != i.end() && exact.value()
+ });
+ }
+ }
+
+ else
+ throw Error("flake registry '%s' has unsupported version %d", path, version);
+
+ } catch (nlohmann::json::exception & e) {
+ warn("cannot parse flake registry '%s': %s", path, e.what());
+ } catch (Error & e) {
+ warn("cannot read flake registry '%s': %s", path, e.what());
+ }
+
+ return registry;
+}
+
+void Registry::write(const Path & path)
+{
+ nlohmann::json arr;
+ for (auto & entry : entries) {
+ nlohmann::json obj;
+ obj["from"] = attrsToJson(entry.from.toAttrs());
+ obj["to"] = attrsToJson(entry.to.toAttrs());
+ if (!entry.extraAttrs.empty())
+ obj["to"].update(attrsToJson(entry.extraAttrs));
+ if (entry.exact)
+ obj["exact"] = true;
+ arr.emplace_back(std::move(obj));
+ }
+
+ nlohmann::json json;
+ json["version"] = 2;
+ json["flakes"] = std::move(arr);
+
+ createDirs(dirOf(path));
+ writeFile(path, json.dump(2));
+}
+
+void Registry::add(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs)
+{
+ entries.emplace_back(
+ Entry {
+ .from = from,
+ .to = to,
+ .extraAttrs = extraAttrs
+ });
+}
+
+void Registry::remove(const Input & input)
+{
+ // FIXME: use C++20 std::erase.
+ for (auto i = entries.begin(); i != entries.end(); )
+ if (i->from == input)
+ i = entries.erase(i);
+ else
+ ++i;
+}
+
+static Path getSystemRegistryPath()
+{
+ return settings.nixConfDir + "/registry.json";
+}
+
+static std::shared_ptr<Registry> getSystemRegistry()
+{
+ static auto systemRegistry =
+ Registry::read(getSystemRegistryPath(), Registry::System);
+ return systemRegistry;
+}
+
+Path getUserRegistryPath()
+{
+ return getHome() + "/.config/nix/registry.json";
+}
+
+std::shared_ptr<Registry> getUserRegistry()
+{
+ static auto userRegistry =
+ Registry::read(getUserRegistryPath(), Registry::User);
+ return userRegistry;
+}
+
+static std::shared_ptr<Registry> flagRegistry =
+ std::make_shared<Registry>(Registry::Flag);
+
+std::shared_ptr<Registry> getFlagRegistry()
+{
+ return flagRegistry;
+}
+
+void overrideRegistry(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs)
+{
+ flagRegistry->add(from, to, extraAttrs);
+}
+
+static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
+{
+ static auto reg = [&]() {
+ auto path = settings.flakeRegistry.get();
+
+ if (!hasPrefix(path, "/")) {
+ auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
+ if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
+ store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
+ path = store->toRealPath(storePath);
+ }
+
+ return Registry::read(path, Registry::Global);
+ }();
+
+ return reg;
+}
+
+Registries getRegistries(ref<Store> store)
+{
+ Registries registries;
+ registries.push_back(getFlagRegistry());
+ registries.push_back(getUserRegistry());
+ registries.push_back(getSystemRegistry());
+ registries.push_back(getGlobalRegistry(store));
+ return registries;
+}
+
+std::pair<Input, Attrs> lookupInRegistries(
+ ref<Store> store,
+ const Input & _input)
+{
+ Attrs extraAttrs;
+ int n = 0;
+ Input input(_input);
+
+ restart:
+
+ n++;
+ if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
+
+ for (auto & registry : getRegistries(store)) {
+ // FIXME: O(n)
+ for (auto & entry : registry->entries) {
+ if (entry.exact) {
+ if (entry.from == input) {
+ input = entry.to;
+ extraAttrs = entry.extraAttrs;
+ goto restart;
+ }
+ } else {
+ if (entry.from.contains(input)) {
+ input = entry.to.applyOverrides(
+ !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
+ !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
+ extraAttrs = entry.extraAttrs;
+ goto restart;
+ }
+ }
+ }
+ }
+
+ if (!input.isDirect())
+ throw Error("cannot find flake '%s' in the flake registries", input.to_string());
+
+ debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string());
+
+ return {input, extraAttrs};
+}
+
+}
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
new file mode 100644
index 000000000..1077af020
--- /dev/null
+++ b/src/libfetchers/registry.hh
@@ -0,0 +1,64 @@
+#pragma once
+
+#include "types.hh"
+#include "fetchers.hh"
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Registry
+{
+ enum RegistryType {
+ Flag = 0,
+ User = 1,
+ System = 2,
+ Global = 3,
+ };
+
+ RegistryType type;
+
+ struct Entry
+ {
+ Input from, to;
+ Attrs extraAttrs;
+ bool exact = false;
+ };
+
+ std::vector<Entry> entries;
+
+ Registry(RegistryType type)
+ : type(type)
+ { }
+
+ static std::shared_ptr<Registry> read(
+ const Path & path, RegistryType type);
+
+ void write(const Path & path);
+
+ void add(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs);
+
+ void remove(const Input & input);
+};
+
+typedef std::vector<std::shared_ptr<Registry>> Registries;
+
+std::shared_ptr<Registry> getUserRegistry();
+
+Path getUserRegistryPath();
+
+Registries getRegistries(ref<Store> store);
+
+void overrideRegistry(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs);
+
+std::pair<Input, Attrs> lookupInRegistries(
+ ref<Store> store,
+ const Input & input);
+
+}
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index f5356f0af..55158cece 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -105,7 +105,7 @@ DownloadFileResult downloadFile(
};
}
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
@@ -120,12 +120,9 @@ Tree downloadTarball(
auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired)
- return Tree {
- .actualPath = store->toRealPath(cached->storePath),
- .storePath = std::move(cached->storePath),
- .info = TreeInfo {
- .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
- },
+ return {
+ Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
+ getIntAttr(cached->infoAttrs, "lastModified")
};
auto res = downloadFile(store, url, name, immutable);
@@ -160,117 +157,72 @@ Tree downloadTarball(
*unpackedStorePath,
immutable);
- return Tree {
- .actualPath = store->toRealPath(*unpackedStorePath),
- .storePath = std::move(*unpackedStorePath),
- .info = TreeInfo {
- .lastModified = lastModified,
- },
+ return {
+ Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
+ lastModified,
};
}
-struct TarballInput : Input
-{
- ParsedURL url;
- std::optional<Hash> hash;
-
- TarballInput(const ParsedURL & url) : url(url)
- { }
-
- std::string type() const override { return "tarball"; }
-
- bool operator ==(const Input & other) const override
- {
- auto other2 = dynamic_cast<const TarballInput *>(&other);
- return
- other2
- && to_string() == other2->to_string()
- && hash == other2->hash;
- }
-
- bool isImmutable() const override
- {
- return hash || narHash;
- }
-
- ParsedURL toURL() const override
- {
- auto url2(url);
- // NAR hashes are preferred over file hashes since tar/zip files
- // don't have a canonical representation.
- if (narHash)
- url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
- else if (hash)
- url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
- return url2;
- }
-
- Attrs toAttrsInternal() const override
- {
- Attrs attrs;
- attrs.emplace("url", url.to_string());
- if (hash)
- attrs.emplace("hash", hash->to_string(SRI, true));
- return attrs;
- }
-
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
- {
- auto tree = downloadTarball(store, url.to_string(), "source", false);
-
- auto input = std::make_shared<TarballInput>(*this);
- input->narHash = store->queryPathInfo(tree.storePath)->narHash;
-
- return {std::move(tree), input};
- }
-};
-
struct TarballInputScheme : InputScheme
{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
{
- if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
+ if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
if (!hasSuffix(url.path, ".zip")
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2"))
- return nullptr;
-
- auto input = std::make_unique<TarballInput>(url);
-
- auto hash = input->url.query.find("hash");
- if (hash != input->url.query.end()) {
- // FIXME: require SRI hash.
- input->hash = Hash(hash->second);
- input->url.query.erase(hash);
- }
-
- auto narHash = input->url.query.find("narHash");
- if (narHash != input->url.query.end()) {
- // FIXME: require SRI hash.
- input->narHash = Hash(narHash->second);
- input->url.query.erase(narHash);
- }
-
+ return {};
+
+ Input input;
+ input.attrs.insert_or_assign("type", "tarball");
+ input.attrs.insert_or_assign("url", url.to_string());
+ auto narHash = url.query.find("narHash");
+ if (narHash != url.query.end())
+ input.attrs.insert_or_assign("narHash", narHash->second);
return input;
}
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "hash")
+ if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
throw Error("unsupported tarball input attribute '%s'", name);
- auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
- if (auto hash = maybeGetStrAttr(attrs, "hash"))
- input->hash = newHashAllowEmpty(*hash, {});
-
+ Input input;
+ input.attrs = attrs;
+ //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
return input;
}
+
+ ParsedURL toURL(const Input & input) override
+ {
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ // NAR hashes are preferred over file hashes since tar/zip files
+ // don't have a canonical representation.
+ if (auto narHash = input.getNarHash())
+ url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
+ /*
+ else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
+ url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
+ */
+ return url;
+ }
+
+ bool hasAllInfo(const Input & input) override
+ {
+ return true;
+ }
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ {
+ auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+ return {std::move(tree), input};
+ }
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc
deleted file mode 100644
index b2d8cfc8d..000000000
--- a/src/libfetchers/tree-info.cc
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "tree-info.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-StorePath TreeInfo::computeStorePath(Store & store) const
-{
- assert(narHash);
- return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
-}
-
-}
diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh
deleted file mode 100644
index 2c7347281..000000000
--- a/src/libfetchers/tree-info.hh
+++ /dev/null
@@ -1,29 +0,0 @@
-#pragma once
-
-#include "path.hh"
-#include "hash.hh"
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix { class Store; }
-
-namespace nix::fetchers {
-
-struct TreeInfo
-{
- Hash narHash;
- std::optional<uint64_t> revCount;
- std::optional<time_t> lastModified;
-
- bool operator ==(const TreeInfo & other) const
- {
- return
- narHash == other.narHash
- && revCount == other.revCount
- && lastModified == other.lastModified;
- }
-
- StorePath computeStorePath(Store & store) const;
-};
-
-}
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 051668e53..09f4cd133 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -34,9 +34,19 @@ MixCommonArgs::MixCommonArgs(const string & programName)
try {
globalConfig.set(name, value);
} catch (UsageError & e) {
- warn(e.what());
+ if (!completions)
+ warn(e.what());
}
}},
+ .completer = [](size_t index, std::string_view prefix) {
+ if (index == 0) {
+ std::map<std::string, Config::SettingInfo> settings;
+ globalConfig.getSettings(settings);
+ for (auto & s : settings)
+ if (hasPrefix(s.first, prefix))
+ completions->insert(s.first);
+ }
+ }
});
addFlag({
diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index c44bb6408..0a7291780 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -1,12 +1,13 @@
#include "loggers.hh"
#include "progress-bar.hh"
+#include "util.hh"
namespace nix {
LogFormat defaultLogFormat = LogFormat::raw;
LogFormat parseLogFormat(const std::string & logFormatStr) {
- if (logFormatStr == "raw")
+ if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS"))
return LogFormat::raw;
else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs;
@@ -26,7 +27,7 @@ Logger * makeDefaultLogger() {
case LogFormat::rawWithLogs:
return makeSimpleLogger(true);
case LogFormat::internalJson:
- return makeJSONLogger(*makeSimpleLogger());
+ return makeJSONLogger(*makeSimpleLogger(true));
case LogFormat::bar:
return makeProgressBar();
case LogFormat::barWithLogs:
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 95a9187de..3f7d99a1d 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -131,7 +131,7 @@ public:
auto state(state_.lock());
std::stringstream oss;
- oss << ei;
+ showErrorInfo(oss, ei, loggerSettings.showTrace.get());
log(*state, ei.level, oss.str());
}
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 1cb422967..2b1f25ca3 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -36,7 +36,7 @@ void printGCWarning()
void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
{
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
@@ -45,7 +45,7 @@ void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & pa
void printMissing(ref<Store> store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
- unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
+ uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
{
if (!willBuild.empty()) {
if (willBuild.size() == 1)
@@ -323,10 +323,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
printError("Try '%1% --help' for more information.", programName);
return 1;
} catch (BaseError & e) {
- if (settings.showTrace && e.prefix() != "")
- printError(e.prefix());
logError(e.info());
- if (e.prefix() != "" && !settings.showTrace)
+ if (e.hasTrace() && !loggerSettings.showTrace.get())
printError("(use '--show-trace' to show detailed location information)");
return e.status;
} catch (std::bad_alloc & e) {
@@ -386,7 +384,7 @@ RunPager::~RunPager()
}
-string showBytes(unsigned long long bytes)
+string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index f558247c0..ffae5d796 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -47,7 +47,7 @@ void printMissing(
void printMissing(ref<Store> store, const StorePathSet & willBuild,
const StorePathSet & willSubstitute, const StorePathSet & unknown,
- unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
+ uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
string getArg(const string & opt,
Strings::iterator & i, const Strings::iterator & end);
@@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt;
/* GC helpers. */
-string showBytes(unsigned long long bytes);
+string showBytes(uint64_t bytes);
struct GCResults;
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 9f52ddafa..30150eeba 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -15,6 +15,7 @@
#include <chrono>
#include <future>
#include <regex>
+#include <fstream>
#include <nlohmann/json.hpp>
@@ -57,6 +58,13 @@ void BinaryCacheStore::init()
}
}
+void BinaryCacheStore::upsertFile(const std::string & path,
+ std::string && data,
+ const std::string & mimeType)
+{
+ upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
+}
+
void BinaryCacheStore::getFile(const std::string & path,
Callback<std::shared_ptr<std::string>> callback) noexcept
{
@@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
}
+AutoCloseFD openFile(const Path & path)
+{
+ auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
+ if (!fd)
+ throw SysError("opening file '%1%'", path);
+ return fd;
+}
+
+struct FileSource : FdSource
+{
+ AutoCloseFD fd2;
+
+ FileSource(const Path & path)
+ : fd2(openFile(path))
+ {
+ fd = fd2.get();
+ }
+};
+
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
- RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+ RepairFlag repair, CheckSigsFlag checkSigs)
{
- // FIXME: See if we can use the original source to reduce memory usage.
- auto nar = make_ref<std::string>(narSource.drain());
+ assert(info.narHash && info.narSize);
+
+ if (!repair && isValidPath(info.path)) {
+ // FIXME: copyNAR -> null sink
+ narSource.drain();
+ return;
+ }
+
+ auto [fdTemp, fnTemp] = createTempFile();
+
+ auto now1 = std::chrono::steady_clock::now();
+
+ /* Read the NAR simultaneously into a CompressionSink+FileSink (to
+ write the compressed NAR to disk), into a HashSink (to get the
+ NAR hash), and into a NarAccessor (to get the NAR listing). */
+ HashSink fileHashSink(htSHA256);
+ std::shared_ptr<FSAccessor> narAccessor;
+ {
+ FdSink fileSink(fdTemp.get());
+ TeeSink teeSink(fileSink, fileHashSink);
+ auto compressionSink = makeCompressionSink(compression, teeSink);
+ TeeSource teeSource(narSource, *compressionSink);
+ narAccessor = makeNarAccessor(teeSource);
+ compressionSink->finish();
+ }
+
+ auto now2 = std::chrono::steady_clock::now();
+
+ auto narInfo = make_ref<NarInfo>(info);
+ narInfo->narSize = info.narSize;
+ narInfo->narHash = info.narHash;
+ narInfo->compression = compression;
+ auto [fileHash, fileSize] = fileHashSink.finish();
+ narInfo->fileHash = fileHash;
+ narInfo->fileSize = fileSize;
+ narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
+ + (compression == "xz" ? ".xz" :
+ compression == "bzip2" ? ".bz2" :
+ compression == "br" ? ".br" :
+ "");
- if (!repair && isValidPath(info.path)) return;
+ auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+ printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
+ printStorePath(narInfo->path), info.narSize,
+ ((1.0 - (double) fileSize / info.narSize) * 100.0),
+ duration);
/* Verify that all references are valid. This may do some .narinfo
reads, but typically they'll already be cached. */
@@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
printStorePath(info.path), printStorePath(ref));
}
- assert(nar->compare(0, narMagic.size(), narMagic) == 0);
-
- auto narInfo = make_ref<NarInfo>(info);
-
- narInfo->narSize = nar->size();
- narInfo->narHash = hashString(htSHA256, *nar);
-
- if (info.narHash && info.narHash != narInfo->narHash)
- throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
-
- auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
-
- auto narAccessor = makeNarAccessor(nar);
-
- if (accessor_)
- accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
-
/* Optionally write a JSON file containing a listing of the
contents of the NAR. */
if (writeNARListing) {
@@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
{
auto res = jsonRoot.placeholder("root");
- listNar(res, narAccessor, "", true);
+ listNar(res, ref<FSAccessor>(narAccessor), "", true);
}
}
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
}
- /* Compress the NAR. */
- narInfo->compression = compression;
- auto now1 = std::chrono::steady_clock::now();
- auto narCompressed = compress(compression, *nar, parallelCompression);
- auto now2 = std::chrono::steady_clock::now();
- narInfo->fileHash = hashString(htSHA256, *narCompressed);
- narInfo->fileSize = narCompressed->size();
-
- auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
- printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
- printStorePath(narInfo->path), narInfo->narSize,
- ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
- duration);
-
- narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
- + (compression == "xz" ? ".xz" :
- compression == "bzip2" ? ".bz2" :
- compression == "br" ? ".br" :
- "");
-
/* Optionally maintain an index of DWARF debug info files
consisting of JSON files named 'debuginfo/<build-id>' that
specify the NAR file and member containing the debug info. */
@@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
/* Atomically write the NAR file. */
if (repair || !fileExists(narInfo->url)) {
stats.narWrite++;
- upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
+ upsertFile(narInfo->url,
+ std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
+ "application/x-nix-nar");
} else
stats.narWriteAverted++;
- stats.narWriteBytes += nar->size();
- stats.narWriteCompressedBytes += narCompressed->size();
+ stats.narWriteBytes += info.narSize;
+ stats.narWriteCompressedBytes += fileSize;
stats.narWriteCompressionTimeMs += duration;
/* Atomically write the NAR info file.*/
@@ -338,7 +372,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
- Hash h;
+ std::optional<Hash> h;
if (method == FileIngestionMethod::Recursive) {
dumpPath(srcPath, sink, filter);
h = hashString(hashAlgo, *sink.s);
@@ -348,10 +382,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
h = hashString(hashAlgo, s);
}
- ValidPathInfo info(makeFixedOutputPath(method, h, name));
+ ValidPathInfo info(makeFixedOutputPath(method, *h, name));
auto source = StringSource { *sink.s };
- addToStore(info, source, repair, CheckSigs, nullptr);
+ addToStore(info, source, repair, CheckSigs);
return std::move(info.path);
}
@@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
StringSink sink;
dumpString(s, sink);
auto source = StringSource { *sink.s };
- addToStore(info, source, repair, CheckSigs, nullptr);
+ addToStore(info, source, repair, CheckSigs);
}
return std::move(info.path);
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 52ef8aa7a..9bcdf5901 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -36,9 +36,13 @@ public:
virtual bool fileExists(const std::string & path) = 0;
virtual void upsertFile(const std::string & path,
- const std::string & data,
+ std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) = 0;
+ void upsertFile(const std::string & path,
+ std::string && data,
+ const std::string & mimeType);
+
/* Note: subclasses must implement at least one of the two
following getFile() methods. */
@@ -75,8 +79,7 @@ public:
{ unsupported("queryPathFromHashPart"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
- RepairFlag repair, CheckSigsFlag checkSigs,
- std::shared_ptr<FSAccessor> accessor) override;
+ RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method, HashType hashAlgo,
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 0c25897f8..4156cd678 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -297,7 +297,7 @@ public:
GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, BuildMode buildMode = bmNormal);
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair);
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
@@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
{
this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
state = &DerivationGoal::haveDerivation;
- name = fmt("building of %s", worker.store.showPaths(drv.outputPaths()));
+ name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store)));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation()
retrySubstitution = false;
for (auto & i : drv->outputs)
- worker.store.addTempRoot(i.second.path);
+ worker.store.addTempRoot(i.second.path(worker.store, drv->name));
/* Check what outputs paths are not already valid. */
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
@@ -1206,7 +1206,7 @@ void DerivationGoal::haveDerivation()
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & i : invalidOutputs)
- addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair));
+ addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair, getDerivationCA(*drv)));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
outputsSubstituted();
@@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure()
StorePathSet outputClosure;
for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
- worker.store.computeFSClosure(i.second.path, outputClosure);
+ worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
}
/* Filter out our own outputs (which we have already checked). */
for (auto & i : drv->outputs)
- outputClosure.erase(i.second.path);
+ outputClosure.erase(i.second.path(worker.store, drv->name));
/* Get all dependencies of this derivation so that we know which
derivation is responsible for which path in the output
@@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure()
if (i.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(i);
for (auto & j : drv.outputs)
- outputsToDrv.insert_or_assign(j.second.path, i);
+ outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
}
/* Check each path (slow!). */
@@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised()
for (auto & j : i.second) {
auto k = inDrv.outputs.find(j);
if (k != inDrv.outputs.end())
- worker.store.computeFSClosure(k->second.path, inputPaths);
+ worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths);
else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
@@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild()
goal can start a build, and if not, the main loop will sleep a
few seconds and then retry this goal. */
PathSet lockFiles;
- for (auto & outPath : drv->outputPaths())
+ for (auto & outPath : drv->outputPaths(worker.store))
lockFiles.insert(worker.store.Store::toRealPath(outPath));
if (!outputLocks.lockPaths(lockFiles, "", false)) {
@@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild()
return;
}
- missingPaths = drv->outputPaths();
+ missingPaths = drv->outputPaths(worker.store);
if (buildMode != bmCheck)
for (auto & i : validPaths) missingPaths.erase(i);
/* If any of the outputs already exist but are not valid, delete
them. */
for (auto & i : drv->outputs) {
- if (worker.store.isValidPath(i.second.path)) continue;
- debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
- deletePath(worker.store.Store::toRealPath(i.second.path));
+ if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
+ debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
+ deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
}
/* Don't do a remote build if the derivation has the attribute
@@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone()
So instead, check if the disk is (nearly) full now. If
so, we don't mark this build as a permanent failure. */
#if HAVE_STATVFS
- unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable
+ uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
struct statvfs st;
if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
- (unsigned long long) st.f_bavail * st.f_bsize < required)
+ (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
if (statvfs(tmpDir.c_str(), &st) == 0 &&
- (unsigned long long) st.f_bavail * st.f_bsize < required)
+ (uint64_t) st.f_bavail * st.f_bsize < required)
diskFull = true;
#endif
@@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone()
fmt("running post-build-hook '%s'", settings.postBuildHook),
Logger::Fields{worker.store.printStorePath(drvPath)});
PushActivity pact(act.id);
- auto outputPaths = drv->outputPaths();
+ auto outputPaths = drv->outputPaths(worker.store);
std::map<std::string, std::string> hookEnvironment = getEnv();
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
@@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
if (j.isDerivation()) {
Derivation drv = worker.store.derivationFromPath(j);
for (auto & k : drv.outputs)
- worker.store.computeFSClosure(k.second.path, paths);
+ worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
}
}
@@ -1950,8 +1950,11 @@ void linkOrCopy(const Path & from, const Path & to)
/* Hard-linking fails if we exceed the maximum link count on a
file (e.g. 32000 of ext3), which is quite possible after a
'nix-store --optimise'. FIXME: actually, why don't we just
- bind-mount in this case? */
- if (errno != EMLINK)
+ bind-mount in this case?
+
+ It can also fail with EPERM in BeegFS v7 and earlier versions
+ which don't allow hard-links to other directories */
+ if (errno != EMLINK && errno != EPERM)
throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to);
}
@@ -2012,7 +2015,7 @@ void DerivationGoal::startBuilder()
/* Substitute output placeholders with the actual output paths. */
for (auto & output : drv->outputs)
- inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path);
+ inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
/* Construct the environment passed to the builder. */
initEnv();
@@ -2038,7 +2041,10 @@ void DerivationGoal::startBuilder()
if (!std::regex_match(fileName, regex))
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
- auto storePath = worker.store.parseStorePath(*i++);
+ auto storePathS = *i++;
+ if (!worker.store.isInStore(storePathS))
+ throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
+ auto storePath = worker.store.toStorePath(storePathS).first;
/* Write closure info to <fileName>. */
writeFile(tmpDir + "/" + fileName,
@@ -2077,7 +2083,7 @@ void DerivationGoal::startBuilder()
for (auto & i : dirsInChroot)
try {
if (worker.store.isInStore(i.second.source))
- worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
+ worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
} catch (InvalidPath & e) {
} catch (Error & e) {
throw Error("while processing 'sandbox-paths': %s", e.what());
@@ -2194,7 +2200,7 @@ void DerivationGoal::startBuilder()
(typically the dependencies of /bin/sh). Throw them
out. */
for (auto & i : drv->outputs)
- dirsInChroot.erase(worker.store.printStorePath(i.second.path));
+ dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
#elif __APPLE__
/* We don't really have any parent prep work to do (yet?)
@@ -2607,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs()
/* Add an "outputs" object containing the output paths. */
nlohmann::json outputs;
for (auto & i : drv->outputs)
- outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites);
+ outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
json["outputs"] = outputs;
/* Handle exportReferencesGraph. */
@@ -2750,8 +2756,8 @@ struct RestrictedStore : public LocalFSStore
void queryReferrers(const StorePath & path, StorePathSet & referrers) override
{ }
- StorePathSet queryDerivationOutputs(const StorePath & path) override
- { throw Error("queryDerivationOutputs"); }
+ OutputPathMap queryDerivationOutputMap(const StorePath & path) override
+ { throw Error("queryDerivationOutputMap"); }
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
{ throw Error("queryPathFromHashPart"); }
@@ -2762,14 +2768,13 @@ struct RestrictedStore : public LocalFSStore
{ throw Error("addToStore"); }
void addToStore(const ValidPathInfo & info, Source & narSource,
- RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
- std::shared_ptr<FSAccessor> accessor = 0) override
+ RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
{
- next->addToStore(info, narSource, repair, checkSigs, accessor);
+ next->addToStore(info, narSource, repair, checkSigs);
goal.addDependency(info.path);
}
- StorePath addToStoreFromDump(const string & dump, const string & name,
+ StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
{
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
@@ -2812,7 +2817,7 @@ struct RestrictedStore : public LocalFSStore
auto drv = derivationFromPath(path.path);
for (auto & output : drv.outputs)
if (wantOutput(output.first, path.outputs))
- newPaths.insert(output.second.path);
+ newPaths.insert(output.second.path(*this, drv.name));
} else if (!goal.isAllowed(path.path))
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
}
@@ -2846,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize) override
+ uint64_t & downloadSize, uint64_t & narSize) override
{
/* This is slightly impure since it leaks information to the
client about what paths will be built/substituted or are
@@ -3574,7 +3579,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
if (store.isStorePath(i))
result.insert(store.parseStorePath(i));
else if (drv.outputs.count(i))
- result.insert(drv.outputs.find(i)->second.path);
+ result.insert(drv.outputs.find(i)->second.path(store, drv.name));
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
}
return result;
@@ -3612,7 +3617,7 @@ void DerivationGoal::registerOutputs()
if (hook) {
bool allValid = true;
for (auto & i : drv->outputs)
- if (!worker.store.isValidPath(i.second.path)) allValid = false;
+ if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
if (allValid) return;
}
@@ -3633,23 +3638,23 @@ void DerivationGoal::registerOutputs()
Nix calls. */
StorePathSet referenceablePaths;
for (auto & p : inputPaths) referenceablePaths.insert(p);
- for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path);
+ for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
for (auto & p : addedPaths) referenceablePaths.insert(p);
/* Check whether the output paths were created, and grep each
output path to determine what other paths it references. Also make all
output paths read-only. */
for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path);
- if (!missingPaths.count(i.second.path)) continue;
+ auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
+ if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
Path actualPath = path;
if (needsHashRewrite()) {
- auto r = redirectedOutputs.find(i.second.path);
+ auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
if (r != redirectedOutputs.end()) {
auto redirected = worker.store.Store::toRealPath(r->second);
if (buildMode == bmRepair
- && redirectedBadOutputs.count(i.second.path)
+ && redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
&& pathExists(redirected))
replaceValidPath(path, redirected);
if (buildMode == bmCheck)
@@ -3718,7 +3723,9 @@ void DerivationGoal::registerOutputs()
if (fixedOutput) {
- if (i.second.hash->method == FileIngestionMethod::Flat) {
+ FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash;
+
+ if (outputHash.method == FileIngestionMethod::Flat) {
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
@@ -3729,13 +3736,13 @@ void DerivationGoal::registerOutputs()
/* Check the hash. In hash mode, move the path produced by
the derivation to its content-addressed location. */
- Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
- ? hashPath(*i.second.hash->hash.type, actualPath).first
- : hashFile(*i.second.hash->hash.type, actualPath);
+ Hash h2 = outputHash.method == FileIngestionMethod::Recursive
+ ? hashPath(outputHash.hash.type, actualPath).first
+ : hashFile(outputHash.hash.type, actualPath);
- auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
+ auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
- if (i.second.hash->hash != h2) {
+ if (outputHash.hash != h2) {
/* Throw an error after registering the path as
valid. */
@@ -3743,7 +3750,7 @@ void DerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
worker.store.printStorePath(dest),
- i.second.hash->hash.to_string(SRI, true),
+ outputHash.hash.to_string(SRI, true),
h2.to_string(SRI, true)));
Path actualDest = worker.store.Store::toRealPath(dest);
@@ -3765,7 +3772,7 @@ void DerivationGoal::registerOutputs()
assert(worker.store.parseStorePath(path) == dest);
ca = FixedOutputHash {
- .method = i.second.hash->method,
+ .method = outputHash.method,
.hash = h2,
};
}
@@ -3780,8 +3787,10 @@ void DerivationGoal::registerOutputs()
time. The hash is stored in the database so that we can
verify later on whether nobody has messed with the store. */
debug("scanning for references inside '%1%'", path);
- HashResult hash;
- auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash));
+ // HashResult hash;
+ auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths));
+ auto references = worker.store.parseStorePathSet(pathSetAndHash.first);
+ HashResult hash = pathSetAndHash.second;
if (buildMode == bmCheck) {
if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
@@ -3889,7 +3898,7 @@ void DerivationGoal::registerOutputs()
/* If this is the first round of several, then move the output out of the way. */
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
for (auto & i : drv->outputs) {
- auto path = worker.store.printStorePath(i.second.path);
+ auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
@@ -3907,7 +3916,7 @@ void DerivationGoal::registerOutputs()
if the result was not determistic? */
if (curRound == nrRounds) {
for (auto & i : drv->outputs) {
- Path prev = worker.store.printStorePath(i.second.path) + checkSuffix;
+ Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
deletePath(prev);
}
}
@@ -4208,9 +4217,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
for (auto & i : drv->outputs) {
if (!wantOutput(i.first, wantedOutputs)) continue;
bool good =
- worker.store.isValidPath(i.second.path) &&
- (!checkHash || worker.pathContentsGood(i.second.path));
- if (good == returnValid) result.insert(i.second.path);
+ worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
+ (!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
+ if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
}
return result;
}
@@ -4267,6 +4276,10 @@ private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
+ /* The path the substituter refers to the path as. This will be
+ * different when the stores have different names. */
+ std::optional<StorePath> subPath;
+
/* The remaining substituters. */
std::list<ref<Store>> subs;
@@ -4300,8 +4313,11 @@ private:
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
+ /* Content address for recomputing store path */
+ std::optional<ContentAddress> ca;
+
public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair);
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
~SubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@@ -4331,10 +4347,11 @@ public:
};
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair)
+SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
+ , ca(ca)
{
state = &SubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
@@ -4409,14 +4426,18 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
- if (sub->storeDir != worker.store.storeDir) {
+ if (ca) {
+ subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca);
+ if (sub->storeDir == worker.store.storeDir)
+ assert(subPath == storePath);
+ } else if (sub->storeDir != worker.store.storeDir) {
tryNext();
return;
}
try {
// FIXME: make async
- info = sub->queryPathInfo(storePath);
+ info = sub->queryPathInfo(subPath ? *subPath : storePath);
} catch (InvalidPath &) {
tryNext();
return;
@@ -4435,6 +4456,19 @@ void SubstitutionGoal::tryNext()
throw;
}
+ if (info->path != storePath) {
+ if (info->isContentAddressed(*sub) && info->references.empty()) {
+ auto info2 = std::make_shared<ValidPathInfo>(*info);
+ info2->path = storePath;
+ info = info2;
+ } else {
+ printError("asked '%s' for '%s' but got '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path));
+ tryNext();
+ return;
+ }
+ }
+
/* Update the total expected download size. */
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info);
@@ -4524,7 +4558,7 @@ void SubstitutionGoal::tryToRun()
PushActivity pact(act.id);
copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()),
- storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
+ subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs);
promise.set_value();
} catch (...) {
@@ -4657,11 +4691,11 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair)
+GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
GoalPtr goal = substitutionGoals[path].lock(); // FIXME
if (!goal) {
- goal = std::make_shared<SubstitutionGoal>(path, *this, repair);
+ goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
substitutionGoals.insert_or_assign(path, goal);
wakeUp(goal);
}
@@ -5003,7 +5037,7 @@ bool Worker::pathContentsGood(const StorePath & path)
if (!pathExists(store.printStorePath(path)))
res = false;
else {
- HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
+ HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
Hash nullHash(htSHA256);
res = info->narHash == nullHash || info->narHash == current.first;
}
@@ -5029,7 +5063,7 @@ void Worker::markContentsGood(const StorePath & path)
static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
{
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh
index 0a37459b0..73c0f5f7f 100644
--- a/src/libstore/builtins/buildenv.hh
+++ b/src/libstore/builtins/buildenv.hh
@@ -9,7 +9,7 @@ struct Package {
Path path;
bool active;
int priority;
- Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
+ Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
};
typedef std::vector<Package> Packages;
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index 1cfe4a46a..6585a480d 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -58,20 +58,6 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
}
};
- /* Try the hashed mirrors first. */
- if (getAttr("outputHashMode") == "flat")
- for (auto hashedMirror : settings.hashedMirrors.get())
- try {
- if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
- auto ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
- auto h = Hash(getAttr("outputHash"), ht);
- fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
- return;
- } catch (Error & e) {
- debug(e.what());
- }
-
- /* Otherwise try the specified URL. */
fetch(mainUrl);
}
diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc
index 3d753836f..f83b98a98 100644
--- a/src/libstore/content-address.cc
+++ b/src/libstore/content-address.cc
@@ -3,7 +3,7 @@
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const {
- return makeFileIngestionPrefix(method) + printHashType(*hash.type);
+ return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
@@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
if (prefix == "text") {
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
Hash hash = Hash(string(hashTypeAndHash));
- if (*hash.type != htSHA256) {
+ if (hash.type != htSHA256) {
throw Error("parseContentAddress: the text hash should have type SHA256");
}
return TextHash { hash };
@@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
return ca ? renderContentAddress(*ca) : "";
}
+Hash getContentAddressHash(const ContentAddress & ca)
+{
+ return std::visit(overloaded {
+ [](TextHash th) {
+ return th.hash;
+ },
+ [](FixedOutputHash fsh) {
+ return fsh.hash;
+ }
+ }, ca);
+}
+
}
diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index ba4797f5b..22a039242 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
+Hash getContentAddressHash(const ContentAddress & ca);
+
}
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 842aef20c..b9a750425 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -78,15 +78,15 @@ struct TunnelLogger : public Logger
if (ei.level > verbosity) return;
std::stringstream oss;
- oss << ei;
+ showErrorInfo(oss, ei, false);
StringSink buf;
- buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
+ buf << STDERR_NEXT << oss.str();
enqueueMsg(*buf.s);
}
/* startWork() means that we're starting an operation for which we
- want to send out stderr to the client. */
+ want to send out stderr to the client. */
void startWork()
{
auto state(state_.lock());
@@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource
}
};
-/* If the NAR archive contains a single file at top-level, then save
- the contents of the file to `s'. Otherwise barf. */
-struct RetrieveRegularNARSink : ParseSink
-{
- bool regular;
- string s;
-
- RetrieveRegularNARSink() : regular(true) { }
-
- void createDirectory(const Path & path)
- {
- regular = false;
- }
-
- void receiveContents(unsigned char * data, unsigned int len)
- {
- s.append((const char *) data, len);
- }
-
- void createSymlink(const Path & path, const string & target)
- {
- regular = false;
- }
-};
-
struct ClientSettings
{
bool keepFailed;
@@ -314,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
- to << hash.to_string(Base16, false);
+ to << hash->to_string(Base16, false);
break;
}
@@ -347,6 +322,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
+ case wopQueryDerivationOutputMap: {
+ auto path = store->parseStorePath(readString(from));
+ logger->startWork();
+ OutputPathMap outputs = store->queryDerivationOutputMap(path);
+ logger->stopWork();
+ writeOutputPathMap(*store, to, outputs);
+ break;
+ }
+
case wopQueryDeriver: {
auto path = store->parseStorePath(readString(from));
logger->startWork();
@@ -366,42 +350,44 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopAddToStore: {
- std::string s, baseName;
+ HashType hashAlgo;
+ std::string baseName;
FileIngestionMethod method;
{
- bool fixed; uint8_t recursive;
- from >> baseName >> fixed /* obsolete */ >> recursive >> s;
+ bool fixed;
+ uint8_t recursive;
+ std::string hashAlgoRaw;
+ from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
method = FileIngestionMethod { recursive };
/* Compatibility hack. */
if (!fixed) {
- s = "sha256";
+ hashAlgoRaw = "sha256";
method = FileIngestionMethod::Recursive;
}
+ hashAlgo = parseHashType(hashAlgoRaw);
}
- HashType hashAlgo = parseHashType(s);
- TeeSource savedNAR(from);
- RetrieveRegularNARSink savedRegular;
+ StringSink saved;
+ TeeSource savedNARSource(from, saved);
+ RetrieveRegularNARSink savedRegular { saved };
if (method == FileIngestionMethod::Recursive) {
/* Get the entire NAR dump from the client and save it to
a string so that we can pass it to
addToStoreFromDump(). */
ParseSink sink; /* null sink; just parse the NAR */
- parseDump(sink, savedNAR);
+ parseDump(sink, savedNARSource);
} else
parseDump(savedRegular, from);
logger->startWork();
if (!savedRegular.regular) throw Error("regular file expected");
- auto path = store->addToStoreFromDump(
- method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
- baseName,
- method,
- hashAlgo);
+ // FIXME: try to stream directly from `from`.
+ StringSource dumpSource { *saved.s };
+ auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
logger->stopWork();
to << store->printStorePath(path);
@@ -433,7 +419,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopImportPaths: {
logger->startWork();
TunnelSource source(from, to);
- auto paths = store->importPaths(source, nullptr,
+ auto paths = store->importPaths(source,
trusted ? NoCheckSigs : CheckSigs);
logger->stopWork();
Strings paths2;
@@ -465,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopBuildDerivation: {
auto drvPath = store->parseStorePath(readString(from));
BasicDerivation drv;
- readDerivation(from, *store, drv);
+ readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
if (!trusted)
@@ -593,7 +579,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto path = store->parseStorePath(readString(from));
logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos({path}, infos);
+ store->querySubstitutablePathInfos({{path, std::nullopt}}, infos);
logger->stopWork();
auto i = infos.find(path);
if (i == infos.end())
@@ -609,10 +595,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQuerySubstitutablePathInfos: {
- auto paths = readStorePaths<StorePathSet>(*store, from);
- logger->startWork();
SubstitutablePathInfos infos;
- store->querySubstitutablePathInfos(paths, infos);
+ StorePathCAMap pathsMap = {};
+ if (GET_PROTOCOL_MINOR(clientVersion) < 22) {
+ auto paths = readStorePaths<StorePathSet>(*store, from);
+ for (auto & path : paths)
+ pathsMap.emplace(path, std::nullopt);
+ } else
+ pathsMap = readStorePathCAMap(*store, from);
+ logger->startWork();
+ store->querySubstitutablePathInfos(pathsMap, infos);
logger->stopWork();
to << infos.size();
for (auto & i : infos) {
@@ -646,7 +638,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
- << info->narHash.to_string(Base16, false);
+ << info->narHash->to_string(Base16, false);
writeStorePaths(*store, to, info->references);
to << info->registrationTime << info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
@@ -717,24 +709,84 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (!trusted)
info.ultimate = false;
- std::string saved;
- std::unique_ptr<Source> source;
- if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
- source = std::make_unique<TunnelSource>(from, to);
- else {
- TeeSink tee(from);
- parseDump(tee, tee.source);
- saved = std::move(*tee.source.data);
- source = std::make_unique<StringSource>(saved);
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
+
+ struct FramedSource : Source
+ {
+ Source & from;
+ bool eof = false;
+ std::vector<unsigned char> pending;
+ size_t pos = 0;
+
+ FramedSource(Source & from) : from(from)
+ { }
+
+ ~FramedSource()
+ {
+ if (!eof) {
+ while (true) {
+ auto n = readInt(from);
+ if (!n) break;
+ std::vector<unsigned char> data(n);
+ from(data.data(), n);
+ }
+ }
+ }
+
+ size_t read(unsigned char * data, size_t len) override
+ {
+ if (eof) throw EndOfFile("reached end of FramedSource");
+
+ if (pos >= pending.size()) {
+ size_t len = readInt(from);
+ if (!len) {
+ eof = true;
+ return 0;
+ }
+ pending = std::vector<unsigned char>(len);
+ pos = 0;
+ from(pending.data(), len);
+ }
+
+ auto n = std::min(len, pending.size() - pos);
+ memcpy(data, pending.data() + pos, n);
+ pos += n;
+ return n;
+ }
+ };
+
+ logger->startWork();
+
+ {
+ FramedSource source(from);
+ store->addToStore(info, source, (RepairFlag) repair,
+ dontCheckSigs ? NoCheckSigs : CheckSigs);
+ }
+
+ logger->stopWork();
}
- logger->startWork();
+ else {
+ std::unique_ptr<Source> source;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
+ source = std::make_unique<TunnelSource>(from, to);
+ else {
+ StringSink saved;
+ TeeSource tee { from, saved };
+ ParseSink ether;
+ parseDump(ether, tee);
+ source = std::make_unique<StringSource>(std::move(*saved.s));
+ }
- // FIXME: race if addToStore doesn't read source?
- store->addToStore(info, *source, (RepairFlag) repair,
- dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
+ logger->startWork();
+
+ // FIXME: race if addToStore doesn't read source?
+ store->addToStore(info, *source, (RepairFlag) repair,
+ dontCheckSigs ? NoCheckSigs : CheckSigs);
+
+ logger->stopWork();
+ }
- logger->stopWork();
break;
}
@@ -744,7 +796,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
targets.push_back(store->parsePathWithOutputs(s));
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
logger->stopWork();
writeStorePaths(*store, to, willBuild);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 42551ef6b..7d0a5abeb 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -4,16 +4,23 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "fs-accessor.hh"
-#include "istringstream_nocopy.hh"
namespace nix {
-const StorePath & BasicDerivation::findOutput(const string & id) const
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
{
- auto i = outputs.find(id);
- if (i == outputs.end())
- throw Error("derivation has no output '%s'", id);
- return i->second.path;
+ return std::visit(overloaded {
+ [](DerivationOutputInputAddressed doi) {
+ return doi.path;
+ },
+ [&](DerivationOutputFixed dof) {
+ return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
+ }
+ }, output);
}
@@ -101,14 +108,13 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
}
-static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
+static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
{
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
expect(str, ","); auto hashAlgo = parseString(str);
expect(str, ","); const auto hash = parseString(str);
expect(str, ")");
- std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@@ -116,23 +122,30 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
hashAlgo = string(hashAlgo, 2);
}
const HashType hashType = parseHashType(hashAlgo);
- fsh = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
- };
- }
- return DerivationOutput {
- .path = std::move(path),
- .hash = std::move(fsh),
- };
+ return DerivationOutput {
+ .output = DerivationOutputFixed {
+ .hash = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash(hash, hashType),
+ },
+ }
+ };
+ } else
+ return DerivationOutput {
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(path),
+ }
+ };
}
-static Derivation parseDerivation(const Store & store, const string & s)
+static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
{
Derivation drv;
- istringstream_nocopy str(s);
+ drv.name = name;
+
+ std::istringstream str(std::move(s));
expect(str, "Derive([");
/* Parse the list of outputs. */
@@ -175,10 +188,10 @@ static Derivation parseDerivation(const Store & store, const string & s)
}
-Derivation readDerivation(const Store & store, const Path & drvPath)
+Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
{
try {
- return parseDerivation(store, readFile(drvPath));
+ return parseDerivation(store, readFile(drvPath), name);
} catch (FormatError & e) {
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
}
@@ -196,7 +209,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
{
auto accessor = getFSAccessor();
try {
- return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
+ return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
}
@@ -264,10 +277,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
for (auto & i : outputs) {
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
- s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
- s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
- s += ','; printUnquotedString(s,
- i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
+ s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
+ if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
+ s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
+ s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
+ } else {
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, "");
+ }
s += ')';
}
@@ -323,7 +340,7 @@ bool BasicDerivation::isFixedOutput() const
{
return outputs.size() == 1 &&
outputs.begin()->first == "out" &&
- outputs.begin()->second.hash;
+ std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output);
}
@@ -355,10 +372,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
/* Return a fixed hash for fixed-output derivations. */
if (drv.isFixedOutput()) {
DerivationOutputs::const_iterator i = drv.outputs.begin();
+ auto hash = std::get<DerivationOutputFixed>(i->second.output);
return hashString(htSHA256, "fixed:out:"
- + i->second.hash->printMethodAlgo() + ":"
- + i->second.hash->hash.to_string(Base16, false) + ":"
- + store.printStorePath(i->second.path));
+ + hash.hash.printMethodAlgo() + ":"
+ + hash.hash.hash.to_string(Base16, false) + ":"
+ + store.printStorePath(i->second.path(store, drv.name)));
}
/* For other derivations, replace the inputs paths with recursive
@@ -392,11 +410,11 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
}
-StorePathSet BasicDerivation::outputPaths() const
+StorePathSet BasicDerivation::outputPaths(const Store & store) const
{
StorePathSet paths;
for (auto & i : outputs)
- paths.insert(i.second.path);
+ paths.insert(i.second.path(store, name));
return paths;
}
@@ -406,7 +424,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
auto hashAlgo = readString(in);
auto hash = readString(in);
- std::optional<FixedOutputHash> fsh;
if (hashAlgo != "") {
auto method = FileIngestionMethod::Flat;
if (string(hashAlgo, 0, 2) == "r:") {
@@ -414,16 +431,20 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
hashAlgo = string(hashAlgo, 2);
}
auto hashType = parseHashType(hashAlgo);
- fsh = FixedOutputHash {
- .method = std::move(method),
- .hash = Hash(hash, hashType),
+ return DerivationOutput {
+ .output = DerivationOutputFixed {
+ .hash = FixedOutputHash {
+ .method = std::move(method),
+ .hash = Hash(hash, hashType),
+ },
+ }
+ };
+ } else
+ return DerivationOutput {
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(path),
+ }
};
- }
-
- return DerivationOutput {
- .path = std::move(path),
- .hash = std::move(fsh),
- };
}
StringSet BasicDerivation::outputNames() const
@@ -435,8 +456,19 @@ StringSet BasicDerivation::outputNames() const
}
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
+std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
+ auto nameWithSuffix = drvPath.name();
+ constexpr std::string_view extension = ".drv";
+ assert(hasSuffix(nameWithSuffix, extension));
+ nameWithSuffix.remove_suffix(extension.size());
+ return nameWithSuffix;
+}
+
+
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
{
+ drv.name = name;
+
drv.outputs.clear();
auto nr = readNum<size_t>(in);
for (size_t n = 0; n < nr; n++) {
@@ -465,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
out << drv.outputs.size();
for (auto & i : drv.outputs) {
out << i.first
- << store.printStorePath(i.second.path);
- if (i.second.hash) {
- out << i.second.hash->printMethodAlgo()
- << i.second.hash->hash.to_string(Base16, false);
+ << store.printStorePath(i.second.path(store, drv.name));
+ if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
+ out << hash->hash.printMethodAlgo()
+ << hash->hash.hash.to_string(Base16, false);
} else {
out << "" << "";
}
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 68c53c1ff..133ffe50e 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -13,10 +13,20 @@ namespace nix {
/* Abstract syntax of derivations. */
-struct DerivationOutput
+struct DerivationOutputInputAddressed
{
StorePath path;
- std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
+};
+
+struct DerivationOutputFixed
+{
+ FixedOutputHash hash; /* hash used for expected hash computation */
+};
+
+struct DerivationOutput
+{
+ std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output;
+ StorePath path(const Store & store, std::string_view drvName) const;
};
typedef std::map<string, DerivationOutput> DerivationOutputs;
@@ -35,24 +45,23 @@ struct BasicDerivation
Path builder;
Strings args;
StringPairs env;
+ std::string name;
BasicDerivation() { }
virtual ~BasicDerivation() { };
- /* Return the path corresponding to the output identifier `id' in
- the given derivation. */
- const StorePath & findOutput(const std::string & id) const;
-
bool isBuiltin() const;
/* Return true iff this is a fixed-output derivation. */
bool isFixedOutput() const;
/* Return the output paths of a derivation. */
- StorePathSet outputPaths() const;
+ StorePathSet outputPaths(const Store & store) const;
/* Return the output names of a derivation. */
StringSet outputNames() const;
+
+ static std::string_view nameFromPath(const StorePath & storePath);
};
struct Derivation : BasicDerivation
@@ -76,7 +85,7 @@ StorePath writeDerivation(ref<Store> store,
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
/* Read a derivation from a file. */
-Derivation readDerivation(const Store & store, const Path & drvPath);
+Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
// FIXME: remove
bool isDerivation(const string & fileName);
@@ -93,7 +102,7 @@ bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source;
struct Sink;
-Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
+Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
std::string hashPlaceholder(const std::string & outputName);
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index 57b7e9590..a0fc22264 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -7,24 +7,6 @@
namespace nix {
-struct HashAndWriteSink : Sink
-{
- Sink & writeSink;
- HashSink hashSink;
- HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
- {
- }
- virtual void operator () (const unsigned char * data, size_t len)
- {
- writeSink(data, len);
- hashSink(data, len);
- }
- Hash currentHash()
- {
- return hashSink.currentHash().first;
- }
-};
-
void Store::exportPaths(const StorePathSet & paths, Sink & sink)
{
auto sorted = topoSortPaths(paths);
@@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
{
auto info = queryPathInfo(path);
- HashAndWriteSink hashAndWriteSink(sink);
+ HashSink hashSink(htSHA256);
+ TeeSink teeSink(sink, hashSink);
- narFromPath(path, hashAndWriteSink);
+ narFromPath(path, teeSink);
/* Refuse to export paths that have changed. This prevents
filesystem corruption from spreading to other machines.
Don't complain if the stored hash is zero (unknown). */
- Hash hash = hashAndWriteSink.currentHash();
- if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
+ Hash hash = hashSink.currentHash().first;
+ if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
- printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
+ printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
- hashAndWriteSink
+ teeSink
<< exportMagic
<< printStorePath(path);
- writeStorePaths(*this, hashAndWriteSink, info->references);
- hashAndWriteSink
+ writeStorePaths(*this, teeSink, info->references);
+ teeSink
<< (info->deriver ? printStorePath(*info->deriver) : "")
<< 0;
}
-StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
+StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
{
StorePaths res;
while (true) {
@@ -77,8 +60,10 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
/* Extract the NAR from the source. */
- TeeSink tee(source);
- parseDump(tee, tee.source);
+ StringSink saved;
+ TeeSource tee { source, saved };
+ ParseSink ether;
+ parseDump(ether, tee);
uint32_t magic = readInt(source);
if (magic != exportMagic)
@@ -94,16 +79,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
if (deriver != "")
info.deriver = parseStorePath(deriver);
- info.narHash = hashString(htSHA256, *tee.source.data);
- info.narSize = tee.source.data->size();
+ info.narHash = hashString(htSHA256, *saved.s);
+ info.narSize = saved.s->size();
// Ignore optional legacy signature.
if (readInt(source) == 1)
readString(source);
// Can't use underlying source, which would have been exhausted
- auto source = StringSource { *tee.source.data };
- addToStore(info, source, NoRepair, checkSigs, accessor);
+ auto source = StringSource { *saved.s };
+ addToStore(info, source, NoRepair, checkSigs);
res.push_back(info.path);
}
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 531b85af8..4149f8155 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -22,6 +22,7 @@
#include <queue>
#include <random>
#include <thread>
+#include <regex>
using namespace std::string_literals;
@@ -56,7 +57,7 @@ struct curlFileTransfer : public FileTransfer
Callback<FileTransferResult> callback;
CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object
- std::string status;
+ std::string statusMsg;
unsigned int attempt = 0;
@@ -123,7 +124,7 @@ struct curlFileTransfer : public FileTransfer
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
- fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
+ fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
} catch (...) {
ignoreException();
}
@@ -144,6 +145,7 @@ struct curlFileTransfer : public FileTransfer
LambdaSink finalSink;
std::shared_ptr<CompressionSink> decompressionSink;
+ std::optional<StringSink> errorSink;
std::exception_ptr writeException;
@@ -153,9 +155,19 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb;
result.bodySize += realSize;
- if (!decompressionSink)
+ if (!decompressionSink) {
decompressionSink = makeDecompressionSink(encoding, finalSink);
+ if (! successfulStatuses.count(getHTTPStatus())) {
+ // In this case we want to construct a TeeSink, to keep
+ // the response around (which we figure won't be big
+ // like an actual download should be) to improve error
+ // messages.
+ errorSink = StringSink { };
+ }
+ }
+ if (errorSink)
+ (*errorSink)((unsigned char *) contents, realSize);
(*decompressionSink)((unsigned char *) contents, realSize);
return realSize;
@@ -175,12 +187,13 @@ struct curlFileTransfer : public FileTransfer
size_t realSize = size * nmemb;
std::string line((char *) contents, realSize);
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
- if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
+ static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
+ std::smatch match;
+ if (std::regex_match(line, match, statusLine)) {
result.etag = "";
- auto ss = tokenizeString<vector<string>>(line, " ");
- status = ss.size() >= 2 ? ss[1] : "";
result.data = std::make_shared<std::string>();
result.bodySize = 0;
+ statusMsg = trim(match[1]);
acceptRanges = false;
encoding = "";
} else {
@@ -194,7 +207,9 @@ struct curlFileTransfer : public FileTransfer
the expected ETag on a 200 response, then shut
down the connection because we already have the
data. */
- if (result.etag == request.expectedETag && status == "200") {
+ long httpStatus = 0;
+ curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+ if (result.etag == request.expectedETag && httpStatus == 200) {
debug(format("shutting down on 200 HTTP response with expected ETag"));
return 0;
}
@@ -408,16 +423,21 @@ struct curlFileTransfer : public FileTransfer
attempt++;
+ std::shared_ptr<std::string> response;
+ if (errorSink)
+ response = errorSink->s;
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
- ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+ ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
: httpStatus != 0
? FileTransferError(err,
- fmt("unable to %s '%s': HTTP error %d",
- request.verb(), request.uri, httpStatus)
+ response,
+ fmt("unable to %s '%s': HTTP error %d ('%s')",
+ request.verb(), request.uri, httpStatus, statusMsg)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
: FileTransferError(err,
+ response,
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@@ -675,7 +695,7 @@ struct curlFileTransfer : public FileTransfer
auto s3Res = s3Helper.getObject(bucketName, key);
FileTransferResult res;
if (!s3Res.data)
- throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+ throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
res.data = s3Res.data;
callback(std::move(res));
#else
@@ -820,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
}
}
+template<typename... Args>
+FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
+ : Error(args...), error(error), response(response)
+{
+ const auto hf = hintfmt(args...);
+ // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
+ // to print different messages for different verbosity levels. For now
+ // we add some heuristics for detecting when we want to show the response.
+ if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
+ err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
+ } else {
+ err.hint = hf;
+ }
+}
+
bool isUri(const string & s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index 11dca2fe0..25ade0add 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -103,10 +103,12 @@ class FileTransferError : public Error
{
public:
FileTransfer::Error error;
+ std::shared_ptr<string> response; // intentionally optional
+
template<typename... Args>
- FileTransferError(FileTransfer::Error error, const Args & ... args)
- : Error(args...), error(error)
- { }
+ FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
+
+ virtual const char* sname() const override { return "FileTransferError"; }
};
bool isUri(const string & s);
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 57fb20845..e74382ed2 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
{
auto foundRoot = [&](const Path & path, const Path & target) {
- auto storePath = maybeParseStorePath(toStorePath(target));
- if (storePath && isValidPath(*storePath))
- roots[std::move(*storePath)].emplace(path);
- else
- printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+ try {
+ auto storePath = toStorePath(target).first;
+ if (isValidPath(storePath))
+ roots[std::move(storePath)].emplace(path);
+ else
+ printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
+ } catch (BadStorePath &) { }
};
try {
@@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
for (auto & [target, links] : unchecked) {
if (!isInStore(target)) continue;
- Path pathS = toStorePath(target);
- if (!isStorePath(pathS)) continue;
- auto path = parseStorePath(pathS);
- if (!isValidPath(path)) continue;
- debug("got additional root '%1%'", pathS);
- if (censor)
- roots[path].insert(censored);
- else
- roots[path].insert(links.begin(), links.end());
+ try {
+ auto path = toStorePath(target).first;
+ if (!isValidPath(path)) continue;
+ debug("got additional root '%1%'", printStorePath(path));
+ if (censor)
+ roots[path].insert(censored);
+ else
+ roots[path].insert(links.begin(), links.end());
+ } catch (BadStorePath &) { }
}
}
@@ -498,7 +500,7 @@ struct LocalStore::GCState
StorePathSet alive;
bool gcKeepOutputs;
bool gcKeepDerivations;
- unsigned long long bytesInvalidated;
+ uint64_t bytesInvalidated;
bool moveToTrash = true;
bool shouldDelete;
GCState(const GCOptions & options, GCResults & results)
@@ -516,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
void LocalStore::deleteGarbage(GCState & state, const Path & path)
{
- unsigned long long bytesFreed;
+ uint64_t bytesFreed;
deletePath(path, bytesFreed);
state.results.bytesFreed += bytesFreed;
}
@@ -526,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
{
checkInterrupt();
- unsigned long long size = 0;
+ uint64_t size = 0;
auto storePath = maybeParseStorePath(path);
if (storePath && isValidPath(*storePath)) {
@@ -685,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
AutoCloseDir dir(opendir(linksDir.c_str()));
if (!dir) throw SysError("opening directory '%1%'", linksDir);
- long long actualSize = 0, unsharedSize = 0;
+ int64_t actualSize = 0, unsharedSize = 0;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
@@ -715,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state)
struct stat st;
if (stat(linksDir.c_str(), &st) == -1)
throw SysError("statting '%1%'", linksDir);
- long long overhead = st.st_blocks * 512ULL;
+ auto overhead = st.st_blocks * 512ULL;
- printInfo(format("note: currently hard linking saves %.2f MiB")
- % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
+ printInfo("note: currently hard linking saves %.2f MiB",
+ ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index bee94cbd8..683fa5196 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -35,7 +35,7 @@ Settings::Settings()
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
, nixManDir(canonPath(NIX_MAN_DIR))
- , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
+ , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
{
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 2fbcafff8..3406a9331 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -196,10 +196,6 @@ public:
/* Whether to lock the Nix client and worker to the same CPU. */
bool lockCPU;
- /* Whether to show a stack trace if Nix evaluation fails. */
- Setting<bool> showTrace{this, false, "show-trace",
- "Whether to show a stack trace on evaluation errors."};
-
Setting<SandboxMode> sandboxMode{this,
#if __linux__
smEnabled
@@ -339,9 +335,6 @@ public:
"setuid/setgid bits or with file capabilities."};
#endif
- Setting<Strings> hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors",
- "A list of servers used by builtins.fetchurl to fetch files by hash."};
-
Setting<uint64_t> minFree{this, 0, "min-free",
"Automatically run the garbage collector when free disk space drops below the specified amount."};
@@ -369,6 +362,12 @@ public:
Setting<bool> warnDirty{this, true, "warn-dirty",
"Whether to warn about dirty Git/Mercurial trees."};
+
+ Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
+ "Maximum size of NARs before spilling them to disk."};
+
+ Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
+ "Path or URI of the global flake registry."};
};
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 451a64785..c1ceb08cf 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -100,11 +100,11 @@ protected:
}
void upsertFile(const std::string & path,
- const std::string & data,
+ std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
auto req = FileTransferRequest(cacheUri + "/" + path);
- req.data = std::make_shared<string>(data); // FIXME: inefficient
+ req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
req.mimeType = mimeType;
try {
getFileTransfer()->upload(req);
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 5657aa593..5d7566121 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
auto s = readString(conn->from);
- info->narHash = s.empty() ? Hash() : Hash(s);
+ info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s};
info->ca = parseContentAddressOpt(readString(conn->from));
info->sigs = readStrings<StringSet>(conn->from);
}
@@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
}
void addToStore(const ValidPathInfo & info, Source & source,
- RepairFlag repair, CheckSigsFlag checkSigs,
- std::shared_ptr<FSAccessor> accessor) override
+ RepairFlag repair, CheckSigsFlag checkSigs) override
{
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
@@ -139,7 +138,7 @@ struct LegacySSHStore : public Store
<< cmdAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
- << info.narHash.to_string(Base16, false);
+ << info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to
<< info.registrationTime
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 48aca478c..87d8334d7 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -31,8 +31,18 @@ protected:
bool fileExists(const std::string & path) override;
void upsertFile(const std::string & path,
- const std::string & data,
- const std::string & mimeType) override;
+ std::shared_ptr<std::basic_iostream<char>> istream,
+ const std::string & mimeType) override
+ {
+ auto path2 = binaryCacheDir + "/" + path;
+ Path tmp = path2 + ".tmp." + std::to_string(getpid());
+ AutoDelete del(tmp, false);
+ StreamToSourceAdapter source(istream);
+ writeFile(tmp, source);
+ if (rename(tmp.c_str(), path2.c_str()))
+ throw SysError("renaming '%1%' to '%2%'", tmp, path2);
+ del.cancel();
+ }
void getFile(const std::string & path, Sink & sink) override
{
@@ -52,7 +62,9 @@ protected:
if (entry.name.size() != 40 ||
!hasSuffix(entry.name, ".narinfo"))
continue;
- paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
+ paths.insert(parseStorePath(
+ storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
+ + "-" + MissingName));
}
return paths;
@@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
BinaryCacheStore::init();
}
-static void atomicWrite(const Path & path, const std::string & s)
-{
- Path tmp = path + ".tmp." + std::to_string(getpid());
- AutoDelete del(tmp, false);
- writeFile(tmp, s);
- if (rename(tmp.c_str(), path.c_str()))
- throw SysError("renaming '%1%' to '%2%'", tmp, path);
- del.cancel();
-}
-
bool LocalBinaryCacheStore::fileExists(const std::string & path)
{
return pathExists(binaryCacheDir + "/" + path);
}
-void LocalBinaryCacheStore::upsertFile(const std::string & path,
- const std::string & data,
- const std::string & mimeType)
-{
- atomicWrite(binaryCacheDir + "/" + path, data);
-}
-
static RegisterStoreImplementation regStore([](
const std::string & uri, const Store::Params & params)
-> std::shared_ptr<Store>
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index dd96d2578..2f1d9663a 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
Path toRealPath(const Path & path)
{
- Path storePath = store->toStorePath(path);
- if (!store->isValidPath(store->parseStorePath(storePath)))
- throw InvalidPath("path '%1%' is not a valid store path", storePath);
+ auto storePath = store->toStorePath(path).first;
+ if (!store->isValidPath(storePath))
+ throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0dfbed9fc..f908f7d67 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
DerivationOutputs::const_iterator out = drv.outputs.find("out");
if (out == drv.outputs.end())
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
-
- check(
- makeFixedOutputPath(
- out->second.hash->method,
- out->second.hash->hash,
- drvName),
- out->second.path, "out");
}
else {
Hash h = hashDerivationModulo(*this, drv, true);
for (auto & i : drv.outputs)
- check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
+ check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
}
}
@@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtRegisterValidPath.use()
(printStorePath(info.path))
- (info.narHash.to_string(Base16, true))
+ (info.narHash->to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
@@ -594,7 +587,7 @@ uint64_t LocalStore::addValidPath(State & state,
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
.exec();
- uint64_t id = sqlite3_last_insert_rowid(state.db);
+ uint64_t id = state.db.getLastInsertedRowId();
/* If this is a derivation, then store the derivation outputs in
the database. This is useful for the garbage collector: it can
@@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmtAddDerivationOutput.use()
(id)
(i.first)
- (printStorePath(i.second.path))
+ (printStorePath(i.second.path(*this, drv.name)))
.exec();
}
}
@@ -686,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmtUpdatePathInfo.use()
(info.narSize, info.narSize != 0)
- (info.narHash.to_string(Base16, true))
+ (info.narHash->to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
@@ -774,17 +767,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
}
-StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
+OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
{
- return retrySQLite<StorePathSet>([&]() {
+ return retrySQLite<OutputPathMap>([&]() {
auto state(_state.lock());
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
(queryValidPathId(*state, path)));
- StorePathSet outputs;
+ OutputPathMap outputs;
while (useQueryDerivationOutputs.next())
- outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
+ outputs.emplace(
+ useQueryDerivationOutputs.getStr(0),
+ parseStorePath(useQueryDerivationOutputs.getStr(1))
+ );
return outputs;
});
@@ -843,20 +839,32 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos)
{
if (!settings.useSubstitutes) return;
for (auto & sub : getDefaultSubstituters()) {
- if (sub->storeDir != storeDir) continue;
for (auto & path : paths) {
- if (infos.count(path)) continue;
- debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path));
+ auto subPath(path.first);
+
+ // recompute store path so that we can use a different store root
+ if (path.second) {
+ subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second);
+ if (sub->storeDir == storeDir)
+ assert(subPath == path.first);
+ if (subPath != path.first)
+ debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri());
+ } else if (sub->storeDir != storeDir) continue;
+
+ debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath));
try {
- auto info = sub->queryPathInfo(path);
+ auto info = sub->queryPathInfo(subPath);
+
+ if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty()))
+ continue;
+
auto narInfo = std::dynamic_pointer_cast<const NarInfo>(
std::shared_ptr<const ValidPathInfo>(info));
- infos.insert_or_assign(path, SubstitutablePathInfo{
+ infos.insert_or_assign(path.first, SubstitutablePathInfo{
info->deriver,
info->references,
narInfo ? narInfo->fileSize : 0,
@@ -897,7 +905,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
StorePathSet paths;
for (auto & i : infos) {
- assert(i.narHash.type == htSHA256);
+ assert(i.narHash && i.narHash->type == htSHA256);
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
else
@@ -959,7 +967,7 @@ const PublicKeys & LocalStore::getPublicKeys()
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
- RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+ RepairFlag repair, CheckSigsFlag checkSigs)
{
if (!info.narHash)
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
@@ -973,7 +981,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
PathLocks outputLock;
- Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
+ auto realPath = Store::toRealPath(info.path);
/* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a
@@ -1010,7 +1018,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
- printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
+ printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
@@ -1030,12 +1038,64 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
}
-StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
+StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
{
- Hash h = hashString(hashAlgo, dump);
+ /* For computing the store path. */
+ auto hashSink = std::make_unique<HashSink>(hashAlgo);
+ TeeSource source { source0, *hashSink };
+
+ /* Read the source path into memory, but only if it's up to
+ narBufferSize bytes. If it's larger, write it to a temporary
+ location in the Nix store. If the subsequently computed
+ destination store path is already valid, we just delete the
+ temporary path. Otherwise, we move it to the destination store
+ path. */
+ bool inMemory = false;
+
+ std::string dump;
+
+ /* Fill out buffer, and decide whether we are working strictly in
+ memory based on whether we break out because the buffer is full
+ or the original source is empty */
+ while (dump.size() < settings.narBufferSize) {
+ auto oldSize = dump.size();
+ constexpr size_t chunkSize = 65536;
+ auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
+ dump.resize(oldSize + want);
+ auto got = 0;
+ try {
+ got = source.read((uint8_t *) dump.data() + oldSize, want);
+ } catch (EndOfFile &) {
+ inMemory = true;
+ break;
+ }
+ dump.resize(oldSize + got);
+ }
+
+ std::unique_ptr<AutoDelete> delTempDir;
+ Path tempPath;
+
+ if (!inMemory) {
+ /* Drain what we pulled so far, and then keep on pulling */
+ StringSource dumpSource { dump };
+ ChainSource bothSource { dumpSource, source };
+
+ auto tempDir = createTempDir(realStoreDir, "add");
+ delTempDir = std::make_unique<AutoDelete>(tempDir);
+ tempPath = tempDir + "/x";
+
+ if (method == FileIngestionMethod::Recursive)
+ restorePath(tempPath, bothSource);
+ else
+ writeFile(tempPath, bothSource);
+
+ dump.clear();
+ }
+
+ auto [hash, size] = hashSink->finish();
- auto dstPath = makeFixedOutputPath(method, h, name);
+ auto dstPath = makeFixedOutputPath(method, hash, name);
addTempRoot(dstPath);
@@ -1044,8 +1104,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
/* The first check above is an optimisation to prevent
unnecessary lock acquisition. */
- Path realPath = realStoreDir + "/";
- realPath += dstPath.to_string();
+ auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
@@ -1055,31 +1114,36 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
autoGC();
- if (method == FileIngestionMethod::Recursive) {
- StringSource source(dump);
- restorePath(realPath, source);
- } else
- writeFile(realPath, dump);
+ if (inMemory) {
+ StringSource dumpSource { dump };
+ /* Restore from the NAR in memory. */
+ if (method == FileIngestionMethod::Recursive)
+ restorePath(realPath, dumpSource);
+ else
+ writeFile(realPath, dumpSource);
+ } else {
+ /* Move the temporary path we restored above. */
+ if (rename(tempPath.c_str(), realPath.c_str()))
+ throw Error("renaming '%s' to '%s'", tempPath, realPath);
+ }
- canonicalisePathMetaData(realPath, -1);
+ /* For computing the nar hash. In recursive SHA-256 mode, this
+ is the same as the store hash, so no need to do it again. */
+ auto narHash = std::pair { hash, size };
+ if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
+ HashSink narSink { htSHA256 };
+ dumpPath(realPath, narSink);
+ narHash = narSink.finish();
+ }
- /* Register the SHA-256 hash of the NAR serialisation of
- the path in the database. We may just have computed it
- above (if called with recursive == true and hashAlgo ==
- sha256); otherwise, compute it here. */
- HashResult hash;
- if (method == FileIngestionMethod::Recursive) {
- hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
- hash.second = dump.size();
- } else
- hash = hashPath(htSHA256, realPath);
+ canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
- optimisePath(realPath); // FIXME: combine with hashPath()
+ optimisePath(realPath);
ValidPathInfo info(dstPath);
- info.narHash = hash.first;
- info.narSize = hash.second;
- info.ca = FixedOutputHash { .method = method, .hash = h };
+ info.narHash = narHash.first;
+ info.narSize = narHash.second;
+ info.ca = FixedOutputHash { .method = method, .hash = hash };
registerValidPath(info);
}
@@ -1090,24 +1154,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
}
-StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
- FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
-{
- Path srcPath(absPath(_srcPath));
-
- /* Read the whole path into memory. This is not a very scalable
- method for very large paths, but `copyPath' is mainly used for
- small files. */
- StringSink sink;
- if (method == FileIngestionMethod::Recursive)
- dumpPath(srcPath, sink, filter);
- else
- sink.s = make_ref<std::string>(readFile(srcPath));
-
- return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
-}
-
-
StorePath LocalStore::addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair)
{
@@ -1118,8 +1164,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
if (repair || !isValidPath(dstPath)) {
- Path realPath = realStoreDir + "/";
- realPath += dstPath.to_string();
+ auto realPath = Store::toRealPath(dstPath);
PathLocks outputLock({realPath});
@@ -1255,9 +1300,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
std::unique_ptr<AbstractHashSink> hashSink;
if (!info->ca || !info->references.count(info->path))
- hashSink = std::make_unique<HashSink>(*info->narHash.type);
+ hashSink = std::make_unique<HashSink>(info->narHash->type);
else
- hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
+ hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
dumpPath(Store::toRealPath(i), *hashSink);
auto current = hashSink->finish();
@@ -1266,7 +1311,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
logError({
.name = "Invalid hash - path modified",
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
- printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
+ printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
});
if (repair) repairPath(i); else errors = true;
} else {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index e17cc45ae..31e6587ac 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -29,8 +29,8 @@ struct Derivation;
struct OptimiseStats
{
unsigned long filesLinked = 0;
- unsigned long long bytesFreed = 0;
- unsigned long long blocksFreed = 0;
+ uint64_t bytesFreed = 0;
+ uint64_t blocksFreed = 0;
};
@@ -133,29 +133,20 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override;
- StorePathSet queryDerivationOutputs(const StorePath & path) override;
+ OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & source,
- RepairFlag repair, CheckSigsFlag checkSigs,
- std::shared_ptr<FSAccessor> accessor) override;
-
- StorePath addToStore(const string & name, const Path & srcPath,
- FileIngestionMethod method, HashType hashAlgo,
- PathFilter & filter, RepairFlag repair) override;
-
- /* Like addToStore(), but the contents of the path are contained
- in `dump', which is either a NAR serialisation (if recursive ==
- true) or simply the contents of a regular file (if recursive ==
- false). */
- StorePath addToStoreFromDump(const string & dump, const string & name,
- FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
+ RepairFlag repair, CheckSigsFlag checkSigs) override;
+
+ StorePath addToStoreFromDump(Source & dump, const string & name,
+ FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override;
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index aec4ed493..d266c8efe 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -61,3 +61,6 @@ $(d)/build.cc:
clean-files += $(d)/schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
+
+$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
+ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index e68edb38c..7f1b62f26 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -108,9 +108,19 @@ void Store::computeFSClosure(const StorePath & startPath,
}
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
+{
+ auto out = drv.outputs.find("out");
+ if (out != drv.outputs.end()) {
+ if (auto v = std::get_if<DerivationOutputFixed>(&out->second.output))
+ return v->hash;
+ }
+ return std::nullopt;
+}
+
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
- unsigned long long & downloadSize_, unsigned long long & narSize_)
+ uint64_t & downloadSize_, uint64_t & narSize_)
{
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
@@ -122,8 +132,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
{
std::unordered_set<std::string> done;
StorePathSet & unknown, & willSubstitute, & willBuild;
- unsigned long long & downloadSize;
- unsigned long long & narSize;
+ uint64_t & downloadSize;
+ uint64_t & narSize;
};
struct DrvState
@@ -157,7 +167,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
auto outPath = parseStorePath(outPathS);
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({outPath}, infos);
+ querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
if (infos.empty()) {
drvState_->lock()->done = true;
@@ -198,8 +208,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
PathSet invalid;
for (auto & j : drv->outputs)
if (wantOutput(j.first, path.outputs)
- && !isValidPath(j.second.path))
- invalid.insert(printStorePath(j.second.path));
+ && !isValidPath(j.second.path(*this, drv->name)))
+ invalid.insert(printStorePath(j.second.path(*this, drv->name)));
if (invalid.empty()) return;
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
@@ -214,7 +224,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
if (isValidPath(path.path)) return;
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({path.path}, infos);
+ querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index ca663d837..59ec164b6 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -18,7 +18,7 @@ struct NarMember
/* If this is a regular file, position of the contents of this
file in the NAR. */
- size_t start = 0, size = 0;
+ uint64_t start = 0, size = 0;
std::string target;
@@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
NarMember root;
- struct NarIndexer : ParseSink, StringSource
+ struct NarIndexer : ParseSink, Source
{
NarAccessor & acc;
+ Source & source;
std::stack<NarMember *> parents;
- std::string currentStart;
bool isExec = false;
- NarIndexer(NarAccessor & acc, const std::string & nar)
- : StringSource(nar), acc(acc)
+ uint64_t pos = 0;
+
+ NarIndexer(NarAccessor & acc, Source & source)
+ : acc(acc), source(source)
{ }
void createMember(const Path & path, NarMember member) {
@@ -77,33 +79,40 @@ struct NarAccessor : public FSAccessor
parents.top()->isExecutable = true;
}
- void preallocateContents(unsigned long long size) override
+ void preallocateContents(uint64_t size) override
{
- currentStart = string(s, pos, 16);
- assert(size <= std::numeric_limits<size_t>::max());
- parents.top()->size = (size_t)size;
+ assert(size <= std::numeric_limits<uint64_t>::max());
+ parents.top()->size = (uint64_t) size;
parents.top()->start = pos;
}
- void receiveContents(unsigned char * data, unsigned int len) override
- {
- // Sanity check
- if (!currentStart.empty()) {
- assert(len < 16 || currentStart == string((char *) data, 16));
- currentStart.clear();
- }
- }
+ void receiveContents(unsigned char * data, size_t len) override
+ { }
void createSymlink(const Path & path, const string & target) override
{
createMember(path,
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
+
+ size_t read(unsigned char * data, size_t len) override
+ {
+ auto n = source.read(data, len);
+ pos += n;
+ return n;
+ }
};
NarAccessor(ref<const std::string> nar) : nar(nar)
{
- NarIndexer indexer(*this, *nar);
+ StringSource source(*nar);
+ NarIndexer indexer(*this, source);
+ parseDump(indexer, indexer);
+ }
+
+ NarAccessor(Source & source)
+ {
+ NarIndexer indexer(*this, source);
parseDump(indexer, indexer);
}
@@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
return make_ref<NarAccessor>(nar);
}
+ref<FSAccessor> makeNarAccessor(Source & source)
+{
+ return make_ref<NarAccessor>(source);
+}
+
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
GetNarBytes getNarBytes)
{
diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh
index 2871199de..8af1272f6 100644
--- a/src/libstore/nar-accessor.hh
+++ b/src/libstore/nar-accessor.hh
@@ -6,10 +6,14 @@
namespace nix {
+struct Source;
+
/* Return an object that provides access to the contents of a NAR
file. */
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
+ref<FSAccessor> makeNarAccessor(Source & source);
+
/* Create a NAR accessor from a NAR listing (in the format produced by
listNar()). The callback getNarBytes(offset, length) is used by the
readFile() method of the accessor to get the contents of files
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 012dea6ea..9ddb9957f 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -230,9 +230,9 @@ public:
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
- (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
+ (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
- (info->narHash.to_string(Base32, true))
+ (info->narHash->to_string(Base32, true))
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index ef04bc859..ca471463c 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -1,6 +1,5 @@
#include "globals.hh"
#include "nar-info.hh"
-#include "store-api.hh"
namespace nix {
@@ -8,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
: ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack
{
auto corrupt = [&]() {
- throw Error("NAR info file '%1%' is corrupt", whence);
+ return Error("NAR info file '%1%' is corrupt", whence);
};
auto parseHashField = [&](const string & s) {
try {
return Hash(s);
} catch (BadHash &) {
- corrupt();
- return Hash(); // never reached
+ throw corrupt();
}
};
@@ -26,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
while (pos < s.size()) {
size_t colon = s.find(':', pos);
- if (colon == std::string::npos) corrupt();
+ if (colon == std::string::npos) throw corrupt();
std::string name(s, pos, colon - pos);
size_t eol = s.find('\n', colon + 2);
- if (eol == std::string::npos) corrupt();
+ if (eol == std::string::npos) throw corrupt();
std::string value(s, colon + 2, eol - colon - 2);
@@ -46,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "FileHash")
fileHash = parseHashField(value);
else if (name == "FileSize") {
- if (!string2Int(value, fileSize)) corrupt();
+ if (!string2Int(value, fileSize)) throw corrupt();
}
else if (name == "NarHash")
narHash = parseHashField(value);
else if (name == "NarSize") {
- if (!string2Int(value, narSize)) corrupt();
+ if (!string2Int(value, narSize)) throw corrupt();
}
else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " ");
- if (!references.empty()) corrupt();
+ if (!references.empty()) throw corrupt();
for (auto & r : refs)
references.insert(StorePath(r));
}
@@ -68,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "Sig")
sigs.insert(value);
else if (name == "CA") {
- if (ca) corrupt();
+ if (ca) throw corrupt();
// FIXME: allow blank ca or require skipping field?
ca = parseContentAddressOpt(value);
}
@@ -78,7 +76,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
if (compression == "") compression = "bzip2";
- if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt();
+ if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt();
}
std::string NarInfo::to_string(const Store & store) const
@@ -88,11 +86,11 @@ std::string NarInfo::to_string(const Store & store) const
res += "URL: " + url + "\n";
assert(compression != "");
res += "Compression: " + compression + "\n";
- assert(fileHash.type == htSHA256);
- res += "FileHash: " + fileHash.to_string(Base32, true) + "\n";
+ assert(fileHash && fileHash->type == htSHA256);
+ res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n";
- assert(narHash.type == htSHA256);
- res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
+ assert(narHash && narHash->type == htSHA256);
+ res += "NarHash: " + narHash->to_string(Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";
diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh
index fa38ccd78..eff19f0ef 100644
--- a/src/libstore/nar-info.hh
+++ b/src/libstore/nar-info.hh
@@ -2,7 +2,7 @@
#include "types.hh"
#include "hash.hh"
-#include "path-info.hh"
+#include "store-api.hh"
namespace nix {
@@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo
{
std::string url;
std::string compression;
- Hash fileHash;
+ std::optional<Hash> fileHash;
uint64_t fileSize = 0;
std::string system;
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index b2b2412a3..e4b4b6213 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -282,7 +282,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats)
}
}
-static string showBytes(unsigned long long bytes)
+static string showBytes(uint64_t bytes)
{
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
}
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index f5dee00a6..2a015ea3c 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -1,17 +1,9 @@
#pragma once
-// TODO many of thes eare not needed.
#include "path.hh"
#include "hash.hh"
#include "content-address.hh"
-#include "serialise.hh"
-#include "crypto.hh"
-#include "lru-cache.hh"
-#include "sync.hh"
-#include "globals.hh"
-#include "config.hh"
-
-#include <limits>
+
#include <string>
#include <optional>
@@ -20,11 +12,24 @@ namespace nix {
class Store;
+
+struct SubstitutablePathInfo
+{
+ std::optional<StorePath> deriver;
+ StorePathSet references;
+ uint64_t downloadSize; /* 0 = unknown or inapplicable */
+ uint64_t narSize; /* 0 = unknown */
+};
+
+typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
+
+
struct ValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
- Hash narHash;
+ // TODO document this
+ std::optional<Hash> narHash;
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
@@ -75,6 +80,12 @@ struct ValidPathInfo
/* Return true iff the path is verifiably content-addressed. */
bool isContentAddressed(const Store & store) const;
+ /* Functions to view references + hasSelfReference as one set, mainly for
+ compatibility's sake. */
+ StorePathSet referencesPossiblyToSelf() const;
+ void insertReferencePossiblyToSelf(StorePath && ref);
+ void setReferencesPossiblyToSelf(StorePathSet && refs);
+
static const size_t maxSigs = std::numeric_limits<size_t>::max();
/* Return the number of signatures on this .narinfo that were
@@ -96,4 +107,5 @@ struct ValidPathInfo
};
typedef list<ValidPathInfo> ValidPathInfos;
+
}
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index b3d8ce95c..dc9dc3897 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -2,8 +2,6 @@
namespace nix {
-MakeError(BadStorePath, Error);
-
static void checkName(std::string_view path, std::string_view name)
{
if (name.empty())
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index 4f79843fe..b03a0f69d 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -62,6 +62,9 @@ public:
typedef std::set<StorePath> StorePathSet;
typedef std::vector<StorePath> StorePaths;
+typedef std::map<string, StorePath> OutputPathMap;
+
+typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index 6cfe393a4..6862b42f0 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -12,30 +12,24 @@
namespace nix {
-static bool cmpGensByNumber(const Generation & a, const Generation & b)
-{
- return a.number < b.number;
-}
-
-
/* Parse a generation name of the format
`<profilename>-<number>-link'. */
-static int parseName(const string & profileName, const string & name)
+static std::optional<GenerationNumber> parseName(const string & profileName, const string & name)
{
- if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1;
+ if (string(name, 0, profileName.size() + 1) != profileName + "-") return {};
string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link");
- if (p == string::npos) return -1;
- int n;
+ if (p == string::npos) return {};
+ unsigned int n;
if (string2Int(string(s, 0, p), n) && n >= 0)
return n;
else
- return -1;
+ return {};
}
-Generations findGenerations(Path profile, int & curGen)
+std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile)
{
Generations gens;
@@ -43,30 +37,34 @@ Generations findGenerations(Path profile, int & curGen)
auto profileName = std::string(baseNameOf(profile));
for (auto & i : readDirectory(profileDir)) {
- int n;
- if ((n = parseName(profileName, i.name)) != -1) {
- Generation gen;
- gen.path = profileDir + "/" + i.name;
- gen.number = n;
+ if (auto n = parseName(profileName, i.name)) {
+ auto path = profileDir + "/" + i.name;
struct stat st;
- if (lstat(gen.path.c_str(), &st) != 0)
- throw SysError("statting '%1%'", gen.path);
- gen.creationTime = st.st_mtime;
- gens.push_back(gen);
+ if (lstat(path.c_str(), &st) != 0)
+ throw SysError("statting '%1%'", path);
+ gens.push_back({
+ .number = *n,
+ .path = path,
+ .creationTime = st.st_mtime
+ });
}
}
- gens.sort(cmpGensByNumber);
+ gens.sort([](const Generation & a, const Generation & b)
+ {
+ return a.number < b.number;
+ });
- curGen = pathExists(profile)
+ return {
+ gens,
+ pathExists(profile)
? parseName(profileName, readLink(profile))
- : -1;
-
- return gens;
+ : std::nullopt
+ };
}
-static void makeName(const Path & profile, unsigned int num,
+static void makeName(const Path & profile, GenerationNumber num,
Path & outLink)
{
Path prefix = (format("%1%-%2%") % profile % num).str();
@@ -78,10 +76,9 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath)
{
/* The new generation number should be higher than old the
previous ones. */
- int dummy;
- Generations gens = findGenerations(profile, dummy);
+ auto [gens, dummy] = findGenerations(profile);
- unsigned int num;
+ GenerationNumber num;
if (gens.size() > 0) {
Generation last = gens.back();
@@ -121,7 +118,7 @@ static void removeFile(const Path & path)
}
-void deleteGeneration(const Path & profile, unsigned int gen)
+void deleteGeneration(const Path & profile, GenerationNumber gen)
{
Path generation;
makeName(profile, gen, generation);
@@ -129,7 +126,7 @@ void deleteGeneration(const Path & profile, unsigned int gen)
}
-static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun)
+static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun)
{
if (dryRun)
printInfo(format("would remove generation %1%") % gen);
@@ -140,31 +137,29 @@ static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRu
}
-void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun)
+void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
- int curGen;
- Generations gens = findGenerations(profile, curGen);
+ auto [gens, curGen] = findGenerations(profile);
- if (gensToDelete.find(curGen) != gensToDelete.end())
+ if (gensToDelete.count(*curGen))
throw Error("cannot delete current generation of profile %1%'", profile);
for (auto & i : gens) {
- if (gensToDelete.find(i.number) == gensToDelete.end()) continue;
+ if (!gensToDelete.count(i.number)) continue;
deleteGeneration2(profile, i.number, dryRun);
}
}
-void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun)
+void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun)
{
PathLocks lock;
lockProfile(lock, profile);
- int curGen;
bool fromCurGen = false;
- Generations gens = findGenerations(profile, curGen);
+ auto [gens, curGen] = findGenerations(profile);
for (auto i = gens.rbegin(); i != gens.rend(); ++i) {
if (i->number == curGen) {
fromCurGen = true;
@@ -186,8 +181,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun)
PathLocks lock;
lockProfile(lock, profile);
- int curGen;
- Generations gens = findGenerations(profile, curGen);
+ auto [gens, curGen] = findGenerations(profile);
for (auto & i : gens)
if (i.number != curGen)
@@ -200,8 +194,7 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun)
PathLocks lock;
lockProfile(lock, profile);
- int curGen;
- Generations gens = findGenerations(profile, curGen);
+ auto [gens, curGen] = findGenerations(profile);
bool canDelete = false;
for (auto i = gens.rbegin(); i != gens.rend(); ++i)
diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh
index 78645d8b6..abe507f0e 100644
--- a/src/libstore/profiles.hh
+++ b/src/libstore/profiles.hh
@@ -9,37 +9,32 @@
namespace nix {
+typedef unsigned int GenerationNumber;
+
struct Generation
{
- int number;
+ GenerationNumber number;
Path path;
time_t creationTime;
- Generation()
- {
- number = -1;
- }
- operator bool() const
- {
- return number != -1;
- }
};
-typedef list<Generation> Generations;
+typedef std::list<Generation> Generations;
/* Returns the list of currently present generations for the specified
- profile, sorted by generation number. */
-Generations findGenerations(Path profile, int & curGen);
+ profile, sorted by generation number. Also returns the number of
+ the current generation. */
+std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile);
class LocalFSStore;
Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath);
-void deleteGeneration(const Path & profile, unsigned int gen);
+void deleteGeneration(const Path & profile, GenerationNumber gen);
-void deleteGenerations(const Path & profile, const std::set<unsigned int> & gensToDelete, bool dryRun);
+void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun);
-void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun);
+void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun);
void deleteOldGenerations(const Path & profile, bool dryRun);
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index a10d536a3..62a3cda61 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -48,13 +48,12 @@ static void search(const unsigned char * s, size_t len,
struct RefScanSink : Sink
{
- HashSink hashSink;
StringSet hashes;
StringSet seen;
string tail;
- RefScanSink() : hashSink(htSHA256) { }
+ RefScanSink() { }
void operator () (const unsigned char * data, size_t len);
};
@@ -62,8 +61,6 @@ struct RefScanSink : Sink
void RefScanSink::operator () (const unsigned char * data, size_t len)
{
- hashSink(data, len);
-
/* It's possible that a reference spans the previous and current
fragment, so search in the concatenation of the tail of the
previous fragment and the start of the current fragment. */
@@ -79,10 +76,12 @@ void RefScanSink::operator () (const unsigned char * data, size_t len)
}
-PathSet scanForReferences(const string & path,
- const PathSet & refs, HashResult & hash)
+std::pair<PathSet, HashResult> scanForReferences(const string & path,
+ const PathSet & refs)
{
- RefScanSink sink;
+ RefScanSink refsSink;
+ HashSink hashSink { htSHA256 };
+ TeeSink sink { refsSink, hashSink };
std::map<string, Path> backMap;
/* For efficiency (and a higher hit rate), just search for the
@@ -97,7 +96,7 @@ PathSet scanForReferences(const string & path,
assert(s.size() == refLength);
assert(backMap.find(s) == backMap.end());
// parseHash(htSHA256, s);
- sink.hashes.insert(s);
+ refsSink.hashes.insert(s);
backMap[s] = i;
}
@@ -106,15 +105,15 @@ PathSet scanForReferences(const string & path,
/* Map the hashes found back to their store paths. */
PathSet found;
- for (auto & i : sink.seen) {
+ for (auto & i : refsSink.seen) {
std::map<string, Path>::iterator j;
if ((j = backMap.find(i)) == backMap.end()) abort();
found.insert(j->second);
}
- hash = sink.hashSink.finish();
+ auto hash = hashSink.finish();
- return found;
+ return std::pair<PathSet, HashResult>(found, hash);
}
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index c38bdd720..598a3203a 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -5,8 +5,7 @@
namespace nix {
-PathSet scanForReferences(const Path & path, const PathSet & refs,
- HashResult & hash);
+std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
struct RewritingSink : Sink
{
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index bd698d781..2d02a181b 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -16,26 +16,26 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir)
createDirs(cacheDir);
}
-Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext)
+Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::string & ext)
{
assert(cacheDir != "");
- return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext);
+ return fmt("%s/%s.%s", cacheDir, hashPart, ext);
}
-void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar,
+void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor)
{
- nars.emplace(storePath, narAccessor);
+ nars.emplace(hashPart, narAccessor);
if (cacheDir != "") {
try {
std::ostringstream str;
JSONPlaceholder jsonRoot(str);
listNar(jsonRoot, narAccessor, "", true);
- writeFile(makeCacheFile(storePath, "ls"), str.str());
+ writeFile(makeCacheFile(hashPart, "ls"), str.str());
/* FIXME: do this asynchronously. */
- writeFile(makeCacheFile(storePath, "nar"), nar);
+ writeFile(makeCacheFile(hashPart, "nar"), nar);
} catch (...) {
ignoreException();
@@ -47,23 +47,22 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
{
auto path = canonPath(path_);
- auto storePath = store->toStorePath(path);
- std::string restPath = std::string(path, storePath.size());
+ auto [storePath, restPath] = store->toStorePath(path);
- if (!store->isValidPath(store->parseStorePath(storePath)))
- throw InvalidPath("path '%1%' is not a valid store path", storePath);
+ if (!store->isValidPath(storePath))
+ throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
- auto i = nars.find(storePath);
+ auto i = nars.find(std::string(storePath.hashPart()));
if (i != nars.end()) return {i->second, restPath};
StringSink sink;
std::string listing;
Path cacheFile;
- if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) {
+ if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) {
try {
- listing = nix::readFile(makeCacheFile(storePath, "ls"));
+ listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls"));
auto narAccessor = makeLazyNarAccessor(listing,
[cacheFile](uint64_t offset, uint64_t length) {
@@ -81,7 +80,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
return buf;
});
- nars.emplace(storePath, narAccessor);
+ nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
@@ -90,15 +89,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
*sink.s = nix::readFile(cacheFile);
auto narAccessor = makeNarAccessor(sink.s);
- nars.emplace(storePath, narAccessor);
+ nars.emplace(storePath.hashPart(), narAccessor);
return {narAccessor, restPath};
} catch (SysError &) { }
}
- store->narFromPath(store->parseStorePath(storePath), sink);
+ store->narFromPath(storePath, sink);
auto narAccessor = makeNarAccessor(sink.s);
- addToCache(storePath, *sink.s, narAccessor);
+ addToCache(storePath.hashPart(), *sink.s, narAccessor);
return {narAccessor, restPath};
}
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 4afb3be95..347cf5764 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -10,7 +10,7 @@ class RemoteFSAccessor : public FSAccessor
{
ref<Store> store;
- std::map<Path, ref<FSAccessor>> nars;
+ std::map<std::string, ref<FSAccessor>> nars;
Path cacheDir;
@@ -18,9 +18,9 @@ class RemoteFSAccessor : public FSAccessor
friend class BinaryCacheStore;
- Path makeCacheFile(const Path & storePath, const std::string & ext);
+ Path makeCacheFile(std::string_view hashPart, const std::string & ext);
- void addToCache(const Path & storePath, const std::string & nar,
+ void addToCache(std::string_view hashPart, const std::string & nar,
ref<FSAccessor> narAccessor);
public:
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index b7cc7a5fc..0c65bcd29 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -8,6 +8,7 @@
#include "derivations.hh"
#include "pool.hh"
#include "finally.hh"
+#include "logging.hh"
#include <sys/types.h>
#include <sys/stat.h>
@@ -38,6 +39,47 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths
out << store.printStorePath(i);
}
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from)
+{
+ StorePathCAMap paths;
+ auto count = readNum<size_t>(from);
+ while (count--)
+ paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from)));
+ return paths;
+}
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths)
+{
+ out << paths.size();
+ for (auto & i : paths) {
+ out << store.printStorePath(i.first);
+ out << renderContentAddress(i.second);
+ }
+}
+
+std::map<string, StorePath> readOutputPathMap(const Store & store, Source & from)
+{
+ std::map<string, StorePath> pathMap;
+ auto rawInput = readStrings<Strings>(from);
+ if (rawInput.size() % 2)
+ throw Error("got an odd number of elements from the daemon when trying to read a output path map");
+ auto curInput = rawInput.begin();
+ while (curInput != rawInput.end()) {
+ auto thisKey = *curInput++;
+ auto thisValue = *curInput++;
+ pathMap.emplace(thisKey, store.parseStorePath(thisValue));
+ }
+ return pathMap;
+}
+
+void writeOutputPathMap(const Store & store, Sink & out, const std::map<string, StorePath> & pathMap)
+{
+ out << 2*pathMap.size();
+ for (auto & i : pathMap) {
+ out << i.first;
+ out << store.printStorePath(i.second);
+ }
+}
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
@@ -197,7 +239,7 @@ void RemoteStore::setOptions(Connection & conn)
overrides.erase(settings.maxSilentTime.name);
overrides.erase(settings.buildCores.name);
overrides.erase(settings.useSubstitutes.name);
- overrides.erase(settings.showTrace.name);
+ overrides.erase(loggerSettings.showTrace.name);
conn.to << overrides.size();
for (auto & i : overrides)
conn.to << i.first << i.second.value;
@@ -308,18 +350,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths)
}
-void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
- SubstitutablePathInfos & infos)
+void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos)
{
- if (paths.empty()) return;
+ if (pathsMap.empty()) return;
auto conn(getConnection());
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
- for (auto & i : paths) {
+ for (auto & i : pathsMap) {
SubstitutablePathInfo info;
- conn->to << wopQuerySubstitutablePathInfo << printStorePath(i);
+ conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first);
conn.processStderr();
unsigned int reply = readInt(conn->from);
if (reply == 0) continue;
@@ -329,13 +370,19 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths,
info.references = readStorePaths<StorePathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
info.narSize = readLongLong(conn->from);
- infos.insert_or_assign(i, std::move(info));
+ infos.insert_or_assign(i.first, std::move(info));
}
} else {
conn->to << wopQuerySubstitutablePathInfos;
- writeStorePaths(*this, conn->to, paths);
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) {
+ StorePathSet paths;
+ for (auto & path : pathsMap)
+ paths.insert(path.first);
+ writeStorePaths(*this, conn->to, paths);
+ } else
+ writeStorePathCAMap(*this, conn->to, pathsMap);
conn.processStderr();
size_t count = readNum<size_t>(conn->from);
for (size_t n = 0; n < count; n++) {
@@ -412,12 +459,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
{
auto conn(getConnection());
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
+ return Store::queryDerivationOutputs(path);
+ }
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return readStorePaths<StorePathSet>(*this, conn->from);
}
+OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path)
+{
+ auto conn(getConnection());
+ conn->to << wopQueryDerivationOutputMap << printStorePath(path);
+ conn.processStderr();
+ return readOutputPathMap(*this, conn->from);
+
+}
+
std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart)
{
auto conn(getConnection());
@@ -430,7 +489,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string &
void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
- RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
+ RepairFlag repair, CheckSigsFlag checkSigs)
{
auto conn(getConnection());
@@ -462,14 +521,89 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << wopAddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
- << info.narHash.to_string(Base16, false);
+ << info.narHash->to_string(Base16, false);
writeStorePaths(*this, conn->to, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
<< repair << !checkSigs;
- bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21;
- if (!tunnel) copyNAR(source, conn->to);
- conn.processStderr(0, tunnel ? &source : nullptr);
+
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) {
+
+ std::exception_ptr ex;
+
+ struct FramedSink : BufferedSink
+ {
+ ConnectionHandle & conn;
+ std::exception_ptr & ex;
+
+ FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex)
+ { }
+
+ ~FramedSink()
+ {
+ try {
+ conn->to << 0;
+ conn->to.flush();
+ } catch (...) {
+ ignoreException();
+ }
+ }
+
+ void write(const unsigned char * data, size_t len) override
+ {
+ /* Don't send more data if the remote has
+ encountered an error. */
+ if (ex) {
+ auto ex2 = ex;
+ ex = nullptr;
+ std::rethrow_exception(ex2);
+ }
+ conn->to << len;
+ conn->to(data, len);
+ };
+ };
+
+ /* Handle log messages / exceptions from the remote on a
+ separate thread. */
+ std::thread stderrThread([&]()
+ {
+ try {
+ conn.processStderr();
+ } catch (...) {
+ ex = std::current_exception();
+ }
+ });
+
+ Finally joinStderrThread([&]()
+ {
+ if (stderrThread.joinable()) {
+ stderrThread.join();
+ if (ex) {
+ try {
+ std::rethrow_exception(ex);
+ } catch (...) {
+ ignoreException();
+ }
+ }
+ }
+ });
+
+ {
+ FramedSink sink(conn, ex);
+ copyNAR(source, sink);
+ sink.flush();
+ }
+
+ stderrThread.join();
+ if (ex)
+ std::rethrow_exception(ex);
+
+ } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) {
+ conn.processStderr(0, &source);
+ } else {
+ copyNAR(source, conn->to);
+ conn.processStderr(0, nullptr);
+ }
}
}
@@ -671,7 +805,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize)
+ uint64_t & downloadSize, uint64_t & narSize)
{
{
auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 80c8e9f11..72d2a6689 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -51,16 +51,16 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override;
+ OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
- void querySubstitutablePathInfos(const StorePathSet & paths,
+ void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
void addToStore(const ValidPathInfo & info, Source & nar,
- RepairFlag repair, CheckSigsFlag checkSigs,
- std::shared_ptr<FSAccessor> accessor) override;
+ RepairFlag repair, CheckSigsFlag checkSigs) override;
StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
@@ -94,7 +94,7 @@ public:
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize) override;
+ uint64_t & downloadSize, uint64_t & narSize) override;
void connect() override;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 427dd48ce..67935f3ba 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -7,7 +7,6 @@
#include "globals.hh"
#include "compression.hh"
#include "filetransfer.hh"
-#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
#include <aws/core/VersionConfig.h>
@@ -262,12 +261,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferManager> transferManager;
std::once_flag transferManagerCreated;
- void uploadFile(const std::string & path, const std::string & data,
+ void uploadFile(const std::string & path,
+ std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType,
const std::string & contentEncoding)
{
- auto stream = std::make_shared<istringstream_nocopy>(data);
-
auto maxThreads = std::thread::hardware_concurrency();
static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor>
@@ -307,7 +305,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<TransferHandle> transferHandle =
transferManager->UploadFile(
- stream, bucketName, path, mimeType,
+ istream, bucketName, path, mimeType,
Aws::Map<Aws::String, Aws::String>(),
nullptr /*, contentEncoding */);
@@ -333,9 +331,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
- auto stream = std::make_shared<istringstream_nocopy>(data);
-
- request.SetBody(stream);
+ request.SetBody(istream);
auto result = checkAws(fmt("AWS error uploading '%s'", path),
s3Helper.client->PutObject(request));
@@ -347,25 +343,31 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1)
.count();
- printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") %
- bucketName % path % data.size() % duration);
+ printInfo("uploaded 's3://%s/%s' in %d ms",
+ bucketName, path, duration);
stats.putTimeMs += duration;
- stats.putBytes += data.size();
stats.put++;
}
- void upsertFile(const std::string & path, const std::string & data,
+ void upsertFile(const std::string & path,
+ std::shared_ptr<std::basic_iostream<char>> istream,
const std::string & mimeType) override
{
+ auto compress = [&](std::string compression)
+ {
+ auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain());
+ return std::make_shared<std::stringstream>(std::move(*compressed));
+ };
+
if (narinfoCompression != "" && hasSuffix(path, ".narinfo"))
- uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression);
+ uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression);
else if (lsCompression != "" && hasSuffix(path, ".ls"))
- uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression);
+ uploadFile(path, compress(lsCompression), mimeType, lsCompression);
else if (logCompression != "" && hasPrefix(path, "log/"))
- uploadFile(path, *compress(logCompression, data), mimeType, logCompression);
+ uploadFile(path, compress(logCompression), mimeType, logCompression);
else
- uploadFile(path, data, mimeType, "");
+ uploadFile(path, istream, mimeType, "");
}
void getFile(const std::string & path, Sink & sink) override
@@ -410,7 +412,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
for (auto object : contents) {
auto & key = object.GetKey();
if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue;
- paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown"));
+ paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName));
}
marker = res.GetNextMarker();
diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh
index 4d43fe4d2..b2b75d498 100644
--- a/src/libstore/s3-binary-cache-store.hh
+++ b/src/libstore/s3-binary-cache-store.hh
@@ -19,7 +19,6 @@ public:
struct Stats
{
std::atomic<uint64_t> put{0};
- std::atomic<uint64_t> putBytes{0};
std::atomic<uint64_t> putTimeMs{0};
std::atomic<uint64_t> get{0};
std::atomic<uint64_t> getBytes{0};
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 76c822c4e..31a1f0cac 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -61,6 +61,11 @@ void SQLite::exec(const std::string & stmt)
});
}
+uint64_t SQLite::getLastInsertedRowId()
+{
+ return sqlite3_last_insert_rowid(db);
+}
+
void SQLiteStmt::create(sqlite3 * db, const string & sql)
{
checkInterrupt();
@@ -95,10 +100,10 @@ SQLiteStmt::Use::~Use()
sqlite3_reset(stmt);
}
-SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
+SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool notNull)
{
if (notNull) {
- if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
+ if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
throwSQLiteError(stmt.db, "binding argument");
} else
bind();
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index dd81ab051..99f0d56ce 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -26,6 +26,8 @@ struct SQLite
void isCache();
void exec(const std::string & stmt);
+
+ uint64_t getLastInsertedRowId();
};
/* RAII wrapper to create and destroy SQLite prepared statements. */
@@ -54,7 +56,7 @@ struct SQLiteStmt
~Use();
/* Bind the next parameter. */
- Use & operator () (const std::string & value, bool notNull = true);
+ Use & operator () (std::string_view value, bool notNull = true);
Use & operator () (const unsigned char * data, size_t len, bool notNull = true);
Use & operator () (int64_t value, bool notNull = true);
Use & bind(); // null
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e4a4ae11e..fb9e30597 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -7,6 +7,7 @@
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
+#include "archive.hh"
#include <future>
@@ -20,15 +21,15 @@ bool Store::isInStore(const Path & path) const
}
-Path Store::toStorePath(const Path & path) const
+std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos)
- return path;
+ return {parseStorePath(path), ""};
else
- return Path(path, 0, slash);
+ return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
}
@@ -41,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
- throw NotInStore("path '%1%' is not in the Nix store", path);
+ throw BadStorePath("path '%1%' is not in the Nix store", path);
return path;
}
StorePath Store::followLinksToStorePath(std::string_view path) const
{
- return parseStorePath(toStorePath(followLinksToStore(path)));
+ return toStorePath(followLinksToStore(path)).first;
}
@@ -192,6 +193,23 @@ StorePath Store::makeFixedOutputPath(
}
}
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references, bool hasSelfReference) const
+{
+ // New template
+ return std::visit(overloaded {
+ [&](TextHash th) {
+ return makeTextPath(name, th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
+ }
+ }, ca);
+}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
@@ -221,6 +239,107 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
+StorePath Store::addToStore(const string & name, const Path & _srcPath,
+ FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
+{
+ Path srcPath(absPath(_srcPath));
+ auto source = sinkToSource([&](Sink & sink) {
+ if (method == FileIngestionMethod::Recursive)
+ dumpPath(srcPath, sink, filter);
+ else
+ readFile(srcPath, sink);
+ });
+ return addToStoreFromDump(*source, name, method, hashAlgo, repair);
+}
+
+
+/*
+The aim of this function is to compute in one pass the correct ValidPathInfo for
+the files that we are trying to add to the store. To accomplish that in one
+pass, given the different kind of inputs that we can take (normal nar archives,
+nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks
+and aliases. Also, since the dataflow is obfuscated by this, we include here a
+graphviz diagram:
+
+digraph graphname {
+ node [shape=box]
+ fileSource -> narSink
+ narSink [style=dashed]
+ narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"]
+ narSink -> narHashSink [style = dashed, label = "else"]
+ unsualHashTee -> narHashSink
+ unsualHashTee -> caHashSink
+ fileSource -> parseSink
+ parseSink [style=dashed]
+ parseSink-> fileSink [style = dashed, label = "Flat"]
+ parseSink -> blank [style = dashed, label = "Recursive"]
+ fileSink -> caHashSink
+}
+*/
+ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
+ FileIngestionMethod method, HashType hashAlgo,
+ std::optional<Hash> expectedCAHash)
+{
+ HashSink narHashSink { htSHA256 };
+ HashSink caHashSink { hashAlgo };
+
+ /* Note that fileSink and unusualHashTee must be mutually exclusive, since
+ they both write to caHashSink. Note that that requisite is currently true
+ because the former is only used in the flat case. */
+ RetrieveRegularNARSink fileSink { caHashSink };
+ TeeSink unusualHashTee { narHashSink, caHashSink };
+
+ auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
+ ? static_cast<Sink &>(unusualHashTee)
+ : narHashSink;
+
+ /* Functionally, this means that fileSource will yield the content of
+ srcPath. The fact that we use scratchpadSink as a temporary buffer here
+ is an implementation detail. */
+ auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
+ dumpPath(srcPath, scratchpadSink);
+ });
+
+ /* tapped provides the same data as fileSource, but we also write all the
+ information to narSink. */
+ TeeSource tapped { *fileSource, narSink };
+
+ ParseSink blank;
+ auto & parseSink = method == FileIngestionMethod::Flat
+ ? fileSink
+ : blank;
+
+ /* The information that flows from tapped (besides being replicated in
+ narSink), is now put in parseSink. */
+ parseDump(parseSink, tapped);
+
+ /* We extract the result of the computation from the sink by calling
+ finish. */
+ auto [narHash, narSize] = narHashSink.finish();
+
+ auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+ ? narHash
+ : caHashSink.finish().first;
+
+ if (expectedCAHash && expectedCAHash != hash)
+ throw Error("hash mismatch for '%s'", srcPath);
+
+ ValidPathInfo info(makeFixedOutputPath(method, hash, name));
+ info.narHash = narHash;
+ info.narSize = narSize;
+ info.ca = FixedOutputHash { .method = method, .hash = hash };
+
+ if (!isValidPath(info.path)) {
+ auto source = sinkToSource([&](Sink & scratchpadSink) {
+ dumpPath(srcPath, scratchpadSink);
+ });
+ addToStore(info, *source);
+ }
+
+ return info;
+}
+
+
Store::Store(const Params & params)
: Config(params)
, state({(size_t) pathInfoCacheSize})
@@ -242,6 +361,16 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
+StorePathSet Store::queryDerivationOutputs(const StorePath & path)
+{
+ auto outputMap = this->queryDerivationOutputMap(path);
+ StorePathSet outputPaths;
+ for (auto & i: outputMap) {
+ outputPaths.emplace(std::move(i.second));
+ }
+ return outputPaths;
+}
+
bool Store::isValidPath(const StorePath & storePath)
{
std::string hashPart(storePath.hashPart());
@@ -306,6 +435,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
}
+static bool goodStorePath(const StorePath & expected, const StorePath & actual)
+{
+ return
+ expected.hashPart() == actual.hashPart()
+ && (expected.name() == Store::MissingName || expected.name() == actual.name());
+}
+
+
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
@@ -333,7 +470,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
- res.second->path != storePath)
+ !goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
@@ -345,7 +482,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
- {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+ {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@@ -358,9 +495,11 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
}
- if (!info || info->path != parseStorePath(storePath)) {
+ auto storePath = parseStorePath(storePathS);
+
+ if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
- throw InvalidPath("path '%s' is not valid", storePath);
+ throw InvalidPath("path '%s' is not valid", storePathS);
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -430,7 +569,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
auto info = queryPathInfo(i);
if (showHash) {
- s += info->narHash.to_string(Base16, false) + "\n";
+ s += info->narHash->to_string(Base16, false) + "\n";
s += (format("%1%\n") % info->narSize).str();
}
@@ -462,7 +601,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
auto info = queryPathInfo(storePath);
jsonPath
- .attr("narHash", info->narHash.to_string(hashBase, true))
+ .attr("narHash", info->narHash->to_string(hashBase, true))
.attr("narSize", info->narSize);
{
@@ -505,7 +644,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
- jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
+ jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)
@@ -581,6 +720,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0;
+ // recompute store path on the chance dstStore does it differently
+ if (info->ca && info->references.empty()) {
+ auto info2 = make_ref<ValidPathInfo>(*info);
+ info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(info->path == info2->path);
+ info = info2;
+ }
+
if (!info->narHash) {
StringSink sink;
srcStore->narFromPath({storePath}, sink);
@@ -616,16 +764,20 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
auto valid = dstStore->queryValidPaths(storePaths, substitute);
- PathSet missing;
+ StorePathSet missing;
+ for (auto & path : storePaths)
+ if (!valid.count(path)) missing.insert(path);
+
+ std::map<StorePath, StorePath> pathsMap;
for (auto & path : storePaths)
- if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
+ pathsMap.insert_or_assign(path, path);
- if (missing.empty()) return;
+ if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@@ -640,30 +792,49 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
ThreadPool pool;
- processGraph<Path>(pool,
- PathSet(missing.begin(), missing.end()),
+ processGraph<StorePath>(pool,
+ StorePathSet(missing.begin(), missing.end()),
+
+ [&](const StorePath & storePath) {
+ auto info = srcStore->queryPathInfo(storePath);
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
- [&](const Path & storePath) {
- if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
+ if (dstStore->isValidPath(storePath)) {
nrDone++;
showProgress();
- return PathSet();
+ return StorePathSet();
}
- auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
-
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
- return srcStore->printStorePathSet(info->references);
+ return info->references;
},
- [&](const Path & storePathS) {
+ [&](const StorePath & storePath) {
checkInterrupt();
- auto storePath = dstStore->parseStorePath(storePathS);
+ auto info = srcStore->queryPathInfo(storePath);
+
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
- if (!dstStore->isValidPath(storePath)) {
+ if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
@@ -672,7 +843,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrFailed++;
if (!settings.keepGoing)
throw e;
- logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
+ logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
showProgress();
return;
}
@@ -681,6 +852,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrDone++;
showProgress();
});
+
+ return pathsMap;
}
@@ -746,7 +919,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
- + narHash.to_string(Base32, true) + ";"
+ + narHash->to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
@@ -757,10 +930,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
@@ -859,12 +1028,20 @@ ref<Store> openStore(const std::string & uri_,
throw Error("don't know how to open Nix store '%s'", uri);
}
+static bool isNonUriPath(const std::string & spec) {
+ return
+ // is not a URL
+ spec.find("://") == std::string::npos
+ // Has at least one path separator, and so isn't a single word that
+ // might be special like "auto"
+ && spec.find("/") != std::string::npos;
+}
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
{
if (uri == "daemon") {
return tDaemon;
- } else if (uri == "local" || hasPrefix(uri, "/")) {
+ } else if (uri == "local" || isNonUriPath(uri)) {
return tLocal;
} else if (uri == "" || uri == "auto") {
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
@@ -888,8 +1065,9 @@ static RegisterStoreImplementation regStore([](
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: {
Store::Params params2 = params;
- if (hasPrefix(uri, "/"))
- params2["root"] = uri;
+ if (isNonUriPath(uri)) {
+ params2["root"] = absPath(uri);
+ }
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
}
default:
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 00b9c385c..e94d975c5 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -32,7 +32,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
-MakeError(NotInStore, Error);
+MakeError(BadStorePath, Error);
class FSAccessor;
@@ -86,7 +86,7 @@ struct GCOptions
StorePathSet pathsToDelete;
/* Stop after at least `maxFreed' bytes have been freed. */
- unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()};
+ uint64_t maxFreed{std::numeric_limits<uint64_t>::max()};
};
@@ -98,21 +98,10 @@ struct GCResults
/* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the
number of bytes that would be or was freed. */
- unsigned long long bytesFreed = 0;
+ uint64_t bytesFreed = 0;
};
-struct SubstitutablePathInfo
-{
- std::optional<StorePath> deriver;
- StorePathSet references;
- unsigned long long downloadSize; /* 0 = unknown or inapplicable */
- unsigned long long narSize; /* 0 = unknown */
-};
-
-typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
-
-
enum BuildMode { bmNormal, bmRepair, bmCheck };
@@ -241,9 +230,9 @@ public:
the Nix store. */
bool isStorePath(std::string_view path) const;
- /* Chop off the parts after the top-level store name, e.g.,
- /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */
- Path toStorePath(const Path & path) const;
+ /* Split a path like /nix/store/<hash>-<name>/<bla> into
+ /nix/store/<hash>-<name> and /<bla>. */
+ std::pair<StorePath, Path> toStorePath(const Path & path) const;
/* Follow symlinks until we end up with a path in the Nix store. */
Path followLinksToStore(std::string_view path) const;
@@ -267,7 +256,11 @@ public:
bool hasSelfReference = false) const;
StorePath makeTextPath(std::string_view name, const Hash & hash,
- const StorePathSet & references) const;
+ const StorePathSet & references = {}) const;
+
+ StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references = {},
+ bool hasSelfReference = false) const;
/* This is the preparatory part of addToStore(); it computes the
store path to which srcPath is to be copied. Returns the store
@@ -308,13 +301,16 @@ public:
SubstituteFlag maybeSubstitute = NoSubstitute);
/* Query the set of all valid paths. Note that for some store
- backends, the name part of store paths may be omitted
- (i.e. you'll get /nix/store/<hash> rather than
+ backends, the name part of store paths may be replaced by 'x'
+ (i.e. you'll get /nix/store/<hash>-x rather than
/nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
- full store path. */
+ full store path. FIXME: should return a set of
+ std::variant<StorePath, HashPart> to get rid of this hack. */
virtual StorePathSet queryAllValidPaths()
{ unsupported("queryAllValidPaths"); }
+ constexpr static const char * MissingName = "x";
+
/* Query information about a valid path. It is permitted to omit
the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const StorePath & path);
@@ -342,8 +338,11 @@ public:
virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; };
/* Query the outputs of the derivation denoted by `path'. */
- virtual StorePathSet queryDerivationOutputs(const StorePath & path)
- { unsupported("queryDerivationOutputs"); }
+ virtual StorePathSet queryDerivationOutputs(const StorePath & path);
+
+ /* Query the mapping outputName=>outputPath for the given derivation */
+ virtual OutputPathMap queryDerivationOutputMap(const StorePath & path)
+ { unsupported("queryDerivationOutputMap"); }
/* Query the full store path given the hash part of a valid store
path, or empty if the path doesn't exist. */
@@ -353,15 +352,15 @@ public:
virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; };
/* Query substitute info (i.e. references, derivers and download
- sizes) of a set of paths. If a path does not have substitute
- info, it's omitted from the resulting ‘infos’ map. */
- virtual void querySubstitutablePathInfos(const StorePathSet & paths,
+ sizes) of a map of paths to their optional ca values. If a path
+ does not have substitute info, it's omitted from the resulting
+ ‘infos’ map. */
+ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) { return; };
/* Import a path into the store. */
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
- RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
- std::shared_ptr<FSAccessor> accessor = 0) = 0;
+ RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
@@ -369,10 +368,21 @@ public:
libutil/archive.hh). */
virtual StorePath addToStore(const string & name, const Path & srcPath,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
- PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0;
+ PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair);
+
+ /* Copy the contents of a path to the store and register the
+ validity the resulting path, using a constant amount of
+ memory. */
+ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath,
+ FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+ std::optional<Hash> expectedCAHash = {});
+ /* Like addToStore(), but the contents of the path are contained
+ in `dump', which is either a NAR serialisation (if recursive ==
+ true) or simply the contents of a regular file (if recursive ==
+ false). */
// FIXME: remove?
- virtual StorePath addToStoreFromDump(const string & dump, const string & name,
+ virtual StorePath addToStoreFromDump(Source & dump, const string & name,
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair)
{
throw Error("addToStoreFromDump() is not supported by this store");
@@ -521,7 +531,7 @@ public:
that will be substituted. */
virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- unsigned long long & downloadSize, unsigned long long & narSize);
+ uint64_t & downloadSize, uint64_t & narSize);
/* Sort a set of paths topologically under the references
relation. If p refers to q, then p precedes q in this list. */
@@ -537,8 +547,7 @@ public:
the Nix store. Optionally, the contents of the NARs are
preloaded into the specified FS accessor to speed up subsequent
access. */
- StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor,
- CheckSigsFlag checkSigs = CheckSigs);
+ StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
struct Stats
{
@@ -652,11 +661,13 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
/* Copy store paths from one store to another. The paths may be copied
- in parallel. They are copied in a topologically sorted order
- (i.e. if A is a reference of B, then A is copied before B), but
- the set of store paths is not automatically closed; use
- copyClosure() for that. */
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+ in parallel. They are copied in a topologically sorted order (i.e.
+ if A is a reference of B, then A is copied before B), but the set
+ of store paths is not automatically closed; use copyClosure() for
+ that. Returns a map of what each path was copied to the dstStore
+ as. */
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
+ const StorePathSet & storePaths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
@@ -755,4 +766,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo(
/* Split URI into protocol+hierarchy part and its parameter set. */
std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri);
+std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv);
+
}
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index ac42457fc..f76b13fb4 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -6,7 +6,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x115
+#define PROTOCOL_VERSION 0x117
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -30,7 +30,7 @@ typedef enum {
wopSetOptions = 19,
wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21,
- wopQueryDerivationOutputs = 22,
+ wopQueryDerivationOutputs = 22, // obsolete
wopQueryAllValidPaths = 23,
wopQueryFailedPaths = 24,
wopClearFailedPaths = 25,
@@ -49,6 +49,7 @@ typedef enum {
wopNarFromPath = 38,
wopAddToStoreNar = 39,
wopQueryMissing = 40,
+ wopQueryDerivationOutputMap = 41,
} WorkerOp;
@@ -69,5 +70,10 @@ template<class T> T readStorePaths(const Store & store, Source & from);
void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths);
+StorePathCAMap readStorePathCAMap(const Store & store, Source & from);
+
+void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths);
+
+void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths);
}
diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
index a38c2d798..ae741f867 100644
--- a/src/libutil/ansicolor.hh
+++ b/src/libutil/ansicolor.hh
@@ -11,7 +11,7 @@ namespace nix {
#define ANSI_GREEN "\e[32;1m"
#define ANSI_YELLOW "\e[33;1m"
#define ANSI_BLUE "\e[34;1m"
-#define ANSI_MAGENTA "\e[35m;1m"
-#define ANSI_CYAN "\e[36m;1m"
+#define ANSI_MAGENTA "\e[35;1m"
+#define ANSI_CYAN "\e[36;1m"
}
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 6a8484705..ce7cf9754 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -150,17 +150,17 @@ static void skipGeneric(Source & source)
static void parseContents(ParseSink & sink, Source & source, const Path & path)
{
- unsigned long long size = readLongLong(source);
+ uint64_t size = readLongLong(source);
sink.preallocateContents(size);
- unsigned long long left = size;
+ uint64_t left = size;
std::vector<unsigned char> buf(65536);
while (left) {
checkInterrupt();
auto n = buf.size();
- if ((unsigned long long)n > left) n = left;
+ if ((uint64_t)n > left) n = left;
source(buf.data(), n);
sink.receiveContents(buf.data(), n);
left -= n;
@@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
names[name] = 0;
}
} else if (s == "node") {
- if (s.empty()) throw badArchive("entry name missing");
+ if (name.empty()) throw badArchive("entry name missing");
parse(sink, source, path + "/" + name);
} else
throw badArchive("unknown field " + s);
@@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
throw SysError("fchmod");
}
- void preallocateContents(unsigned long long len)
+ void preallocateContents(uint64_t len)
{
#if HAVE_POSIX_FALLOCATE
if (len) {
@@ -338,7 +338,7 @@ struct RestoreSink : ParseSink
#endif
}
- void receiveContents(unsigned char * data, unsigned int len)
+ void receiveContents(unsigned char * data, size_t len)
{
writeFull(fd.get(), data, len);
}
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 768fe2536..5665732d2 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -57,17 +57,35 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
- virtual void preallocateContents(unsigned long long size) { };
- virtual void receiveContents(unsigned char * data, unsigned int len) { };
+ virtual void preallocateContents(uint64_t size) { };
+ virtual void receiveContents(unsigned char * data, size_t len) { };
virtual void createSymlink(const Path & path, const string & target) { };
};
-struct TeeSink : ParseSink
+/* If the NAR archive contains a single file at top-level, then save
+ the contents of the file to `s'. Otherwise barf. */
+struct RetrieveRegularNARSink : ParseSink
{
- TeeSource source;
+ bool regular = true;
+ Sink & sink;
- TeeSink(Source & source) : source(source) { }
+ RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
+
+ void createDirectory(const Path & path)
+ {
+ regular = false;
+ }
+
+ void receiveContents(unsigned char * data, size_t len)
+ {
+ sink(data, len);
+ }
+
+ void createSymlink(const Path & path, const string & target)
+ {
+ regular = false;
+ }
};
void parseDump(ParseSink & sink, Source & source);
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 1f0e4f803..986c5d1cd 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -1,6 +1,8 @@
#include "args.hh"
#include "hash.hh"
+#include <glob.h>
+
namespace nix {
void Args::addFlag(Flag && flag_)
@@ -13,6 +15,20 @@ void Args::addFlag(Flag && flag_)
if (flag->shortName) shortFlags[flag->shortName] = flag;
}
+bool pathCompletions = false;
+std::shared_ptr<std::set<std::string>> completions;
+
+std::string completionMarker = "___COMPLETE___";
+
+std::optional<std::string> needsCompletion(std::string_view s)
+{
+ if (!completions) return {};
+ auto i = s.find(completionMarker);
+ if (i != std::string::npos)
+ return std::string(s.begin(), i);
+ return {};
+}
+
void Args::parseCmdline(const Strings & _cmdline)
{
Strings pendingArgs;
@@ -20,6 +36,14 @@ void Args::parseCmdline(const Strings & _cmdline)
Strings cmdline(_cmdline);
+ if (auto s = getEnv("NIX_GET_COMPLETIONS")) {
+ size_t n = std::stoi(*s);
+ assert(n > 0 && n <= cmdline.size());
+ *std::next(cmdline.begin(), n - 1) += completionMarker;
+ completions = std::make_shared<decltype(completions)::element_type>();
+ verbosity = lvlError;
+ }
+
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
auto arg = *pos;
@@ -63,7 +87,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
for (auto & exp : expectedArgs) {
std::cout << renderLabels({exp.label});
// FIXME: handle arity > 1
- if (exp.arity == 0) std::cout << "...";
+ if (exp.handler.arity == ArityAny) std::cout << "...";
if (exp.optional) std::cout << "?";
}
std::cout << "\n";
@@ -99,18 +123,32 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
auto process = [&](const std::string & name, const Flag & flag) -> bool {
++pos;
std::vector<std::string> args;
+ bool anyCompleted = false;
for (size_t n = 0 ; n < flag.handler.arity; ++n) {
if (pos == end) {
if (flag.handler.arity == ArityAny) break;
throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
}
+ if (flag.completer)
+ if (auto prefix = needsCompletion(*pos)) {
+ anyCompleted = true;
+ flag.completer(n, *prefix);
+ }
args.push_back(*pos++);
}
- flag.handler.fun(std::move(args));
+ if (!anyCompleted)
+ flag.handler.fun(std::move(args));
return true;
};
if (string(*pos, 0, 2) == "--") {
+ if (auto prefix = needsCompletion(*pos)) {
+ for (auto & [name, flag] : longFlags) {
+ if (!hiddenCategories.count(flag->category)
+ && hasPrefix(name, std::string(*prefix, 2)))
+ completions->insert("--" + name);
+ }
+ }
auto i = longFlags.find(string(*pos, 2));
if (i == longFlags.end()) return false;
return process("--" + i->first, *i->second);
@@ -123,6 +161,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
return process(std::string("-") + c, *i->second);
}
+ if (auto prefix = needsCompletion(*pos)) {
+ if (prefix == "-") {
+ completions->insert("--");
+ for (auto & [flag, _] : shortFlags)
+ completions->insert(std::string("-") + flag);
+ }
+ }
+
return false;
}
@@ -138,12 +184,17 @@ bool Args::processArgs(const Strings & args, bool finish)
bool res = false;
- if ((exp.arity == 0 && finish) ||
- (exp.arity > 0 && args.size() == exp.arity))
+ if ((exp.handler.arity == ArityAny && finish) ||
+ (exp.handler.arity != ArityAny && args.size() == exp.handler.arity))
{
std::vector<std::string> ss;
- for (auto & s : args) ss.push_back(s);
- exp.handler(std::move(ss));
+ for (const auto &[n, s] : enumerate(args)) {
+ ss.push_back(s);
+ if (exp.completer)
+ if (auto prefix = needsCompletion(s))
+ exp.completer(n, *prefix);
+ }
+ exp.handler.fun(ss);
expectedArgs.pop_front();
res = true;
}
@@ -154,6 +205,13 @@ bool Args::processArgs(const Strings & args, bool finish)
return res;
}
+static void hashTypeCompleter(size_t index, std::string_view prefix)
+{
+ for (auto & type : hashTypes)
+ if (hasPrefix(type, prefix))
+ completions->insert(type);
+}
+
Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
{
return Flag {
@@ -162,7 +220,8 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
.labels = {"hash-algo"},
.handler = {[ht](std::string s) {
*ht = parseHashType(s);
- }}
+ }},
+ .completer = hashTypeCompleter
};
}
@@ -174,10 +233,42 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
.labels = {"hash-algo"},
.handler = {[oht](std::string s) {
*oht = std::optional<HashType> { parseHashType(s) };
- }}
+ }},
+ .completer = hashTypeCompleter
};
}
+static void completePath(std::string_view prefix, bool onlyDirs)
+{
+ pathCompletions = true;
+ glob_t globbuf;
+ int flags = GLOB_NOESCAPE | GLOB_TILDE;
+ #ifdef GLOB_ONLYDIR
+ if (onlyDirs)
+ flags |= GLOB_ONLYDIR;
+ #endif
+ if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) {
+ for (size_t i = 0; i < globbuf.gl_pathc; ++i) {
+ if (onlyDirs) {
+ auto st = lstat(globbuf.gl_pathv[i]);
+ if (!S_ISDIR(st.st_mode)) continue;
+ }
+ completions->insert(globbuf.gl_pathv[i]);
+ }
+ globfree(&globbuf);
+ }
+}
+
+void completePath(size_t, std::string_view prefix)
+{
+ completePath(prefix, false);
+}
+
+void completeDir(size_t, std::string_view prefix)
+{
+ completePath(prefix, true);
+}
+
Strings argvToStrings(int argc, char * * argv)
{
Strings args;
@@ -225,18 +316,26 @@ void Command::printHelp(const string & programName, std::ostream & out)
MultiCommand::MultiCommand(const Commands & commands)
: commands(commands)
{
- expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
- assert(!command);
- auto cmd = ss[0];
- if (auto alias = get(deprecatedAliases, cmd)) {
- warn("'%s' is a deprecated alias for '%s'", cmd, *alias);
- cmd = *alias;
- }
- auto i = commands.find(cmd);
- if (i == commands.end())
- throw UsageError("'%s' is not a recognised command", cmd);
- command = {cmd, i->second()};
- }});
+ expectArgs({
+ .label = "command",
+ .optional = true,
+ .handler = {[=](std::string s) {
+ assert(!command);
+ if (auto alias = get(deprecatedAliases, s)) {
+ warn("'%s' is a deprecated alias for '%s'", s, *alias);
+ s = *alias;
+ }
+ if (auto prefix = needsCompletion(s)) {
+ for (auto & [name, command] : commands)
+ if (hasPrefix(name, *prefix))
+ completions->insert(name);
+ }
+ auto i = commands.find(s);
+ if (i == commands.end())
+ throw UsageError("'%s' is not a recognised command", s);
+ command = {s, i->second()};
+ }}
+ });
categories[Command::catDefault] = "Available commands";
}
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 433d26bba..97a517344 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -8,8 +8,6 @@
namespace nix {
-MakeError(UsageError, Error);
-
enum HashType : char;
class Args
@@ -28,61 +26,67 @@ protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max();
+ struct Handler
+ {
+ std::function<void(std::vector<std::string>)> fun;
+ size_t arity;
+
+ Handler() {}
+
+ Handler(std::function<void(std::vector<std::string>)> && fun)
+ : fun(std::move(fun))
+ , arity(ArityAny)
+ { }
+
+ Handler(std::function<void()> && handler)
+ : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
+ , arity(0)
+ { }
+
+ Handler(std::function<void(std::string)> && handler)
+ : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+ handler(std::move(ss[0]));
+ })
+ , arity(1)
+ { }
+
+ Handler(std::function<void(std::string, std::string)> && handler)
+ : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+ handler(std::move(ss[0]), std::move(ss[1]));
+ })
+ , arity(2)
+ { }
+
+ Handler(std::vector<std::string> * dest)
+ : fun([=](std::vector<std::string> ss) { *dest = ss; })
+ , arity(ArityAny)
+ { }
+
+ template<class T>
+ Handler(T * dest)
+ : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+ , arity(1)
+ { }
+
+ template<class T>
+ Handler(T * dest, const T & val)
+ : fun([=](std::vector<std::string> ss) { *dest = val; })
+ , arity(0)
+ { }
+ };
+
/* Flags. */
struct Flag
{
typedef std::shared_ptr<Flag> ptr;
- struct Handler
- {
- std::function<void(std::vector<std::string>)> fun;
- size_t arity;
-
- Handler() {}
-
- Handler(std::function<void(std::vector<std::string>)> && fun)
- : fun(std::move(fun))
- , arity(ArityAny)
- { }
-
- Handler(std::function<void()> && handler)
- : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
- , arity(0)
- { }
-
- Handler(std::function<void(std::string)> && handler)
- : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
- handler(std::move(ss[0]));
- })
- , arity(1)
- { }
-
- Handler(std::function<void(std::string, std::string)> && handler)
- : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
- handler(std::move(ss[0]), std::move(ss[1]));
- })
- , arity(2)
- { }
-
- template<class T>
- Handler(T * dest)
- : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
- , arity(1)
- { }
-
- template<class T>
- Handler(T * dest, const T & val)
- : fun([=](std::vector<std::string> ss) { *dest = val; })
- , arity(0)
- { }
- };
-
std::string longName;
char shortName = 0;
std::string description;
std::string category;
Strings labels;
Handler handler;
+ std::function<void(size_t, std::string_view)> completer;
static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
@@ -99,9 +103,9 @@ protected:
struct ExpectedArg
{
std::string label;
- size_t arity; // 0 = any
- bool optional;
- std::function<void(std::vector<std::string>)> handler;
+ bool optional = false;
+ Handler handler;
+ std::function<void(size_t, std::string_view)> completer;
};
std::list<ExpectedArg> expectedArgs;
@@ -175,20 +179,28 @@ public:
});
}
+ void expectArgs(ExpectedArg && arg)
+ {
+ expectedArgs.emplace_back(std::move(arg));
+ }
+
/* Expect a string argument. */
void expectArg(const std::string & label, string * dest, bool optional = false)
{
- expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) {
- *dest = ss[0];
- }});
+ expectArgs({
+ .label = label,
+ .optional = true,
+ .handler = {dest}
+ });
}
/* Expect 0 or more arguments. */
void expectArgs(const std::string & label, std::vector<std::string> * dest)
{
- expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
- *dest = std::move(ss);
- }});
+ expectArgs({
+ .label = label,
+ .handler = {dest}
+ });
}
friend class MultiCommand;
@@ -259,4 +271,13 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
void printTable(std::ostream & out, const Table2 & table);
+extern std::shared_ptr<std::set<std::string>> completions;
+extern bool pathCompletions;
+
+std::optional<std::string> needsCompletion(std::string_view s);
+
+void completePath(size_t, std::string_view prefix);
+
+void completeDir(size_t, std::string_view prefix);
+
}
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 0fad9ae42..fd6f69b7f 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -7,14 +7,11 @@
namespace nix {
-
const std::string nativeSystem = SYSTEM;
-// addPrefix is used for show-trace. Strings added with addPrefix
-// will print ahead of the error itself.
-BaseError & BaseError::addPrefix(const FormatOrString & fs)
+BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
{
- prefix_ = fs.s + prefix_;
+ err.traces.push_front(Trace { .pos = e, .hint = hint});
return *this;
}
@@ -28,7 +25,7 @@ const string& BaseError::calcWhat() const
err.name = sname();
std::ostringstream oss;
- oss << err;
+ showErrorInfo(oss, err, false);
what_ = oss.str();
return *what_;
@@ -56,28 +53,114 @@ string showErrPos(const ErrPos &errPos)
}
}
-// if nixCode contains lines of code, print them to the ostream, indicating the error column.
-void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixCode)
+std::optional<LinesOfCode> getCodeLines(const ErrPos &errPos)
+{
+ if (errPos.line <= 0)
+ return std::nullopt;
+
+ if (errPos.origin == foFile) {
+ LinesOfCode loc;
+ try {
+ AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
+ if (!fd) {
+ logError(SysError("opening file '%1%'", errPos.file).info());
+ return std::nullopt;
+ }
+ else
+ {
+ // count the newlines.
+ int count = 0;
+ string line;
+ int pl = errPos.line - 1;
+ do
+ {
+ line = readLine(fd.get());
+ ++count;
+ if (count < pl)
+ {
+ ;
+ }
+ else if (count == pl) {
+ loc.prevLineOfCode = line;
+ } else if (count == pl + 1) {
+ loc.errLineOfCode = line;
+ } else if (count == pl + 2) {
+ loc.nextLineOfCode = line;
+ break;
+ }
+ } while (true);
+ return loc;
+ }
+ }
+ catch (EndOfFile &eof) {
+ if (loc.errLineOfCode.has_value())
+ return loc;
+ else
+ return std::nullopt;
+ }
+ catch (std::exception &e) {
+ printError("error reading nix file: %s\n%s", errPos.file, e.what());
+ return std::nullopt;
+ }
+ } else {
+ std::istringstream iss(errPos.file);
+ // count the newlines.
+ int count = 0;
+ string line;
+ int pl = errPos.line - 1;
+
+ LinesOfCode loc;
+
+ do
+ {
+ std::getline(iss, line);
+ ++count;
+ if (count < pl)
+ {
+ ;
+ }
+ else if (count == pl) {
+ loc.prevLineOfCode = line;
+ } else if (count == pl + 1) {
+ loc.errLineOfCode = line;
+ } else if (count == pl + 2) {
+ loc.nextLineOfCode = line;
+ break;
+ }
+
+ if (!iss.good())
+ break;
+ } while (true);
+
+ return loc;
+ }
+}
+
+// print lines of code to the ostream, indicating the error column.
+void printCodeLines(std::ostream &out,
+ const string &prefix,
+ const ErrPos &errPos,
+ const LinesOfCode &loc)
{
// previous line of code.
- if (nixCode.prevLineOfCode.has_value()) {
+ if (loc.prevLineOfCode.has_value()) {
out << std::endl
<< fmt("%1% %|2$5d|| %3%",
- prefix,
- (nixCode.errPos.line - 1),
- *nixCode.prevLineOfCode);
+ prefix,
+ (errPos.line - 1),
+ *loc.prevLineOfCode);
}
- if (nixCode.errLineOfCode.has_value()) {
+ if (loc.errLineOfCode.has_value()) {
// line of code containing the error.
out << std::endl
<< fmt("%1% %|2$5d|| %3%",
- prefix,
- (nixCode.errPos.line),
- *nixCode.errLineOfCode);
+ prefix,
+ (errPos.line),
+ *loc.errLineOfCode);
// error arrows for the column range.
- if (nixCode.errPos.column > 0) {
- int start = nixCode.errPos.column;
+ if (errPos.column > 0) {
+ int start = errPos.column;
std::string spaces;
for (int i = 0; i < start; ++i) {
spaces.append(" ");
@@ -87,23 +170,49 @@ void printCodeLines(std::ostream &out, const string &prefix, const NixCode &nixC
out << std::endl
<< fmt("%1% |%2%" ANSI_RED "%3%" ANSI_NORMAL,
- prefix,
- spaces,
- arrows);
+ prefix,
+ spaces,
+ arrows);
}
}
// next line of code.
- if (nixCode.nextLineOfCode.has_value()) {
+ if (loc.nextLineOfCode.has_value()) {
out << std::endl
<< fmt("%1% %|2$5d|| %3%",
- prefix,
- (nixCode.errPos.line + 1),
- *nixCode.nextLineOfCode);
+ prefix,
+ (errPos.line + 1),
+ *loc.nextLineOfCode);
}
}
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
+void printAtPos(const string &prefix, const ErrPos &pos, std::ostream &out)
+{
+ if (pos)
+ {
+ switch (pos.origin) {
+ case foFile: {
+ out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+ ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
+ break;
+ }
+ case foString: {
+ out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+ ANSI_BLUE << " from string" << ANSI_NORMAL;
+ break;
+ }
+ case foStdin: {
+ out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
+ ANSI_BLUE << " from stdin" << ANSI_NORMAL;
+ break;
+ }
+ default:
+ throw Error("invalid FileOrigin in errPos");
+ }
+ }
+}
+
+std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace)
{
auto errwidth = std::max<size_t>(getWindowSize().second, 20);
string prefix = "";
@@ -158,8 +267,12 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
}
}
- auto ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
- auto dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+ auto ndl = prefix.length()
+ + filterANSIEscapes(levelString, true).length()
+ + 7
+ + einfo.name.length()
+ + einfo.programName.value_or("").length();
+ auto dashwidth = std::max<int>(errwidth - ndl, 3);
std::string dashes(dashwidth, '-');
@@ -179,16 +292,9 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
einfo.programName.value_or(""));
bool nl = false; // intersperse newline between sections.
- if (einfo.nixCode.has_value()) {
- if (einfo.nixCode->errPos.file != "") {
- // filename, line, column.
- out << std::endl << fmt("%1%in file: " ANSI_BLUE "%2% %3%" ANSI_NORMAL,
- prefix,
- einfo.nixCode->errPos.file,
- showErrPos(einfo.nixCode->errPos));
- } else {
- out << std::endl << fmt("%1%from command line argument", prefix);
- }
+ if (einfo.errPos.has_value() && (*einfo.errPos)) {
+ out << prefix << std::endl;
+ printAtPos(prefix, *einfo.errPos, out);
nl = true;
}
@@ -200,12 +306,16 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
nl = true;
}
- // lines of code.
- if (einfo.nixCode.has_value() && einfo.nixCode->errLineOfCode.has_value()) {
- if (nl)
- out << std::endl << prefix;
- printCodeLines(out, prefix, *einfo.nixCode);
- nl = true;
+ if (einfo.errPos.has_value() && (*einfo.errPos)) {
+ auto loc = getCodeLines(*einfo.errPos);
+
+ // lines of code.
+ if (loc.has_value()) {
+ if (nl)
+ out << std::endl << prefix;
+ printCodeLines(out, prefix, *einfo.errPos, *loc);
+ nl = true;
+ }
}
// hint
@@ -216,6 +326,54 @@ std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo)
nl = true;
}
+ // traces
+ if (showTrace && !einfo.traces.empty())
+ {
+ const string tracetitle(" show-trace ");
+
+ int fill = errwidth - tracetitle.length();
+ int lw = 0;
+ int rw = 0;
+ const int min_dashes = 3;
+ if (fill > min_dashes * 2) {
+ if (fill % 2 != 0) {
+ lw = fill / 2;
+ rw = lw + 1;
+ }
+ else
+ {
+ lw = rw = fill / 2;
+ }
+ }
+ else
+ lw = rw = min_dashes;
+
+ if (nl)
+ out << std::endl << prefix;
+
+ out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
+
+ for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
+ {
+ out << std::endl << prefix;
+ out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
+
+ if (iter->pos.has_value() && (*iter->pos)) {
+ auto pos = iter->pos.value();
+ out << std::endl << prefix;
+ printAtPos(prefix, pos, out);
+
+ auto loc = getCodeLines(pos);
+ if (loc.has_value())
+ {
+ out << std::endl << prefix;
+ printCodeLines(out, prefix, pos, *loc);
+ out << std::endl << prefix;
+ }
+ }
+ }
+ }
+
return out;
}
}
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 1e6102ce1..0daaf3be2 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -1,8 +1,8 @@
#pragma once
-
#include "ref.hh"
#include "types.hh"
+#include "fmt.hh"
#include <cstring>
#include <list>
@@ -10,7 +10,9 @@
#include <map>
#include <optional>
-#include "fmt.hh"
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
/* Before 4.7, gcc's std::exception uses empty throw() specifiers for
* its (virtual) destructor and what() in c++11 mode, in violation of spec
@@ -25,20 +27,20 @@ namespace nix {
/*
-This file defines two main structs/classes used in nix error handling.
+ This file defines two main structs/classes used in nix error handling.
-ErrorInfo provides a standard payload of error information, with conversion to string
-happening in the logger rather than at the call site.
+ ErrorInfo provides a standard payload of error information, with conversion to string
+ happening in the logger rather than at the call site.
-BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
-an ErrorInfo.
+ BaseError is the ancestor of nix specific exceptions (and Interrupted), and contains
+ an ErrorInfo.
-ErrorInfo structs are sent to the logger as part of an exception, or directly with the
-logError or logWarning macros.
+ ErrorInfo structs are sent to the logger as part of an exception, or directly with the
+ logError or logWarning macros.
-See the error-demo.cc program for usage examples.
+ See the error-demo.cc program for usage examples.
-*/
+ */
typedef enum {
lvlError = 0,
@@ -50,11 +52,25 @@ typedef enum {
lvlVomit
} Verbosity;
+typedef enum {
+ foFile,
+ foStdin,
+ foString
+} FileOrigin;
+
+// the lines of code surrounding an error.
+struct LinesOfCode {
+ std::optional<string> prevLineOfCode;
+ std::optional<string> errLineOfCode;
+ std::optional<string> nextLineOfCode;
+};
+
// ErrPos indicates the location of an error in a nix file.
struct ErrPos {
int line = 0;
int column = 0;
string file;
+ FileOrigin origin;
operator bool() const
{
@@ -65,9 +81,14 @@ struct ErrPos {
template <class P>
ErrPos& operator=(const P &pos)
{
+ origin = pos.origin;
line = pos.line;
column = pos.column;
- file = pos.file;
+ // is file symbol null?
+ if (pos.file.set())
+ file = pos.file;
+ else
+ file = "";
return *this;
}
@@ -78,11 +99,9 @@ struct ErrPos {
}
};
-struct NixCode {
- ErrPos errPos;
- std::optional<string> prevLineOfCode;
- std::optional<string> errLineOfCode;
- std::optional<string> nextLineOfCode;
+struct Trace {
+ std::optional<ErrPos> pos;
+ hintformat hint;
};
struct ErrorInfo {
@@ -90,19 +109,19 @@ struct ErrorInfo {
string name;
string description;
std::optional<hintformat> hint;
- std::optional<NixCode> nixCode;
+ std::optional<ErrPos> errPos;
+ std::list<Trace> traces;
static std::optional<string> programName;
};
-std::ostream& operator<<(std::ostream &out, const ErrorInfo &einfo);
+std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace);
/* BaseError should generally not be caught, as it has Interrupted as
a subclass. Catch Error instead. */
class BaseError : public std::exception
{
protected:
- string prefix_; // used for location traces etc.
mutable ErrorInfo err;
mutable std::optional<string> what_;
@@ -113,23 +132,23 @@ public:
template<typename... Args>
BaseError(unsigned int status, const Args & ... args)
- : err { .level = lvlError,
- .hint = hintfmt(args...)
- }
+ : err {.level = lvlError,
+ .hint = hintfmt(args...)
+ }
, status(status)
{ }
template<typename... Args>
BaseError(const std::string & fs, const Args & ... args)
- : err { .level = lvlError,
- .hint = hintfmt(fs, args...)
- }
+ : err {.level = lvlError,
+ .hint = hintfmt(fs, args...)
+ }
{ }
BaseError(hintformat hint)
- : err { .level = lvlError,
- .hint = hint
- }
+ : err {.level = lvlError,
+ .hint = hint
+ }
{ }
BaseError(ErrorInfo && e)
@@ -150,10 +169,17 @@ public:
#endif
const string & msg() const { return calcWhat(); }
- const string & prefix() const { return prefix_; }
- BaseError & addPrefix(const FormatOrString & fs);
-
const ErrorInfo & info() { calcWhat(); return err; }
+
+ template<typename... Args>
+ BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args)
+ {
+ return addTrace(e, hintfmt(fs, args...));
+ }
+
+ BaseError & addTrace(std::optional<ErrPos> e, hintformat hint);
+
+ bool hasTrace() const { return !err.traces.empty(); }
};
#define MakeError(newClass, superClass) \
@@ -165,6 +191,7 @@ public:
}
MakeError(Error, BaseError);
+MakeError(UsageError, Error);
class SysError : public Error
{
@@ -173,7 +200,7 @@ public:
template<typename... Args>
SysError(const Args & ... args)
- :Error("")
+ : Error("")
{
errNo = errno;
auto hf = hintfmt(args...);
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 01fae3044..2b0390da4 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -8,7 +8,6 @@
#include "hash.hh"
#include "archive.hh"
#include "util.hh"
-#include "istringstream_nocopy.hh"
#include <sys/types.h>
#include <sys/stat.h>
@@ -16,16 +15,22 @@
namespace nix {
+static size_t regularHashSize(HashType type) {
+ switch (type) {
+ case htMD5: return md5HashSize;
+ case htSHA1: return sha1HashSize;
+ case htSHA256: return sha256HashSize;
+ case htSHA512: return sha512HashSize;
+ }
+ abort();
+}
+
+std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
+
void Hash::init()
{
- assert(type);
- switch (*type) {
- case htMD5: hashSize = md5HashSize; break;
- case htSHA1: hashSize = sha1HashSize; break;
- case htSHA256: hashSize = sha256HashSize; break;
- case htSHA512: hashSize = sha512HashSize; break;
- }
+ hashSize = regularHashSize(type);
assert(hashSize <= maxHashSize);
memset(hash, 0, maxHashSize);
}
@@ -106,17 +111,11 @@ string printHash16or32(const Hash & hash)
}
-HashType assertInitHashType(const Hash & h)
-{
- assert(h.type);
- return *h.type;
-}
-
std::string Hash::to_string(Base base, bool includeType) const
{
std::string s;
if (base == SRI || includeType) {
- s += printHashType(assertInitHashType(*this));
+ s += printHashType(type);
s += base == SRI ? '-' : ':';
}
switch (base) {
@@ -137,60 +136,66 @@ std::string Hash::to_string(Base base, bool includeType) const
Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { }
-Hash::Hash(std::string_view s, std::optional<HashType> type)
- : type(type)
+Hash::Hash(std::string_view original, std::optional<HashType> optType)
{
+ auto rest = original;
+
size_t pos = 0;
bool isSRI = false;
- auto sep = s.find(':');
- if (sep == string::npos) {
- sep = s.find('-');
- if (sep != string::npos) {
- isSRI = true;
- } else if (! type)
- throw BadHash("hash '%s' does not include a type", s);
+ // Parse the has type before the separater, if there was one.
+ std::optional<HashType> optParsedType;
+ {
+ auto sep = rest.find(':');
+ if (sep == std::string_view::npos) {
+ sep = rest.find('-');
+ if (sep != std::string_view::npos)
+ isSRI = true;
+ }
+ if (sep != std::string_view::npos) {
+ auto hashRaw = rest.substr(0, sep);
+ optParsedType = parseHashType(hashRaw);
+ rest = rest.substr(sep + 1);
+ }
}
- if (sep != string::npos) {
- string hts = string(s, 0, sep);
- this->type = parseHashType(hts);
- if (!this->type)
- throw BadHash("unknown hash type '%s'", hts);
- if (type && type != this->type)
- throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type));
- pos = sep + 1;
+ // Either the string or user must provide the type, if they both do they
+ // must agree.
+ if (!optParsedType && !optType) {
+ throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
+ } else {
+ this->type = optParsedType ? *optParsedType : *optType;
+ if (optParsedType && optType && *optParsedType != *optType)
+ throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
}
init();
- size_t size = s.size() - pos;
-
- if (!isSRI && size == base16Len()) {
+ if (!isSRI && rest.size() == base16Len()) {
auto parseHexDigit = [&](char c) {
if (c >= '0' && c <= '9') return c - '0';
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
- throw BadHash("invalid base-16 hash '%s'", s);
+ throw BadHash("invalid base-16 hash '%s'", original);
};
for (unsigned int i = 0; i < hashSize; i++) {
hash[i] =
- parseHexDigit(s[pos + i * 2]) << 4
- | parseHexDigit(s[pos + i * 2 + 1]);
+ parseHexDigit(rest[pos + i * 2]) << 4
+ | parseHexDigit(rest[pos + i * 2 + 1]);
}
}
- else if (!isSRI && size == base32Len()) {
+ else if (!isSRI && rest.size() == base32Len()) {
- for (unsigned int n = 0; n < size; ++n) {
- char c = s[pos + size - n - 1];
+ for (unsigned int n = 0; n < rest.size(); ++n) {
+ char c = rest[rest.size() - n - 1];
unsigned char digit;
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break;
if (digit >= 32)
- throw BadHash("invalid base-32 hash '%s'", s);
+ throw BadHash("invalid base-32 hash '%s'", original);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
@@ -200,21 +205,21 @@ Hash::Hash(std::string_view s, std::optional<HashType> type)
hash[i + 1] |= digit >> (8 - j);
} else {
if (digit >> (8 - j))
- throw BadHash("invalid base-32 hash '%s'", s);
+ throw BadHash("invalid base-32 hash '%s'", original);
}
}
}
- else if (isSRI || size == base64Len()) {
- auto d = base64Decode(s.substr(pos));
+ else if (isSRI || rest.size() == base64Len()) {
+ auto d = base64Decode(rest);
if (d.size() != hashSize)
- throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
+ throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original);
assert(hashSize);
memcpy(hash, d.data(), hashSize);
}
else
- throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type));
+ throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
}
Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
@@ -267,7 +272,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
}
-Hash hashString(HashType ht, const string & s)
+Hash hashString(HashType ht, std::string_view s)
{
Ctx ctx;
Hash hash(ht);
@@ -334,7 +339,7 @@ HashResult hashPath(
Hash compressHash(const Hash & hash, unsigned int newSize)
{
- Hash h;
+ Hash h(hash.type);
h.hashSize = newSize;
for (unsigned int i = 0; i < hash.hashSize; ++i)
h.hash[i % newSize] ^= hash.hash[i];
@@ -342,7 +347,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
}
-std::optional<HashType> parseHashTypeOpt(const string & s)
+std::optional<HashType> parseHashTypeOpt(std::string_view s)
{
if (s == "md5") return htMD5;
else if (s == "sha1") return htSHA1;
@@ -351,7 +356,7 @@ std::optional<HashType> parseHashTypeOpt(const string & s)
else return std::optional<HashType> {};
}
-HashType parseHashType(const string & s)
+HashType parseHashType(std::string_view s)
{
auto opt_h = parseHashTypeOpt(s);
if (opt_h)
@@ -370,7 +375,7 @@ string printHashType(HashType ht)
default:
// illegal hash type enum value internally, as opposed to external input
// which should be validated with nice error message.
- abort();
+ assert(false);
}
}
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 23259dced..abcd58f24 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -18,6 +18,8 @@ const int sha1HashSize = 20;
const int sha256HashSize = 32;
const int sha512HashSize = 64;
+extern std::set<std::string> hashTypes;
+
extern const string base32Chars;
enum Base : int { Base64, Base32, Base16, SRI };
@@ -25,14 +27,11 @@ enum Base : int { Base64, Base32, Base16, SRI };
struct Hash
{
- static const unsigned int maxHashSize = 64;
- unsigned int hashSize = 0;
- unsigned char hash[maxHashSize] = {};
-
- std::optional<HashType> type = {};
+ constexpr static size_t maxHashSize = 64;
+ size_t hashSize = 0;
+ uint8_t hash[maxHashSize] = {};
- /* Create an unset hash object. */
- Hash() { };
+ HashType type;
/* Create a zero-filled hash object. */
Hash(HashType type) : type(type) { init(); };
@@ -105,14 +104,14 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht);
string printHash16or32(const Hash & hash);
/* Compute the hash of the given string. */
-Hash hashString(HashType ht, const string & s);
+Hash hashString(HashType ht, std::string_view s);
/* Compute the hash of the given file. */
Hash hashFile(HashType ht, const Path & path);
/* Compute the hash of the given path. The hash is defined as
(essentially) hashString(ht, dumpPath(path)). */
-typedef std::pair<Hash, unsigned long long> HashResult;
+typedef std::pair<Hash, uint64_t> HashResult;
HashResult hashPath(HashType ht, const Path & path,
PathFilter & filter = defaultPathFilter);
@@ -121,9 +120,10 @@ HashResult hashPath(HashType ht, const Path & path,
Hash compressHash(const Hash & hash, unsigned int newSize);
/* Parse a string representing a hash type. */
-HashType parseHashType(const string & s);
+HashType parseHashType(std::string_view s);
+
/* Will return nothing on parse error */
-std::optional<HashType> parseHashTypeOpt(const string & s);
+std::optional<HashType> parseHashTypeOpt(std::string_view s);
/* And the reverse. */
string printHashType(HashType ht);
@@ -141,7 +141,7 @@ class HashSink : public BufferedSink, public AbstractHashSink
private:
HashType ht;
Ctx * ctx;
- unsigned long long bytes;
+ uint64_t bytes;
public:
HashSink(HashType ht);
diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh
deleted file mode 100644
index f7beac578..000000000
--- a/src/libutil/istringstream_nocopy.hh
+++ /dev/null
@@ -1,92 +0,0 @@
-/* This file provides a variant of std::istringstream that doesn't
- copy its string argument. This is useful for large strings. The
- caller must ensure that the string object is not destroyed while
- it's referenced by this object. */
-
-#pragma once
-
-#include <string>
-#include <iostream>
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
-{
-public:
- typedef std::basic_string<CharT, Traits, Allocator> string_type;
-
- typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
-
- typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
-
- typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
-
- typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
-
-private:
- const string_type & s;
-
- off_type off;
-
-public:
- basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
- {
- }
-
-private:
- pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
- {
- if (which & std::ios_base::in) {
- this->off = dir == std::ios_base::beg
- ? off
- : (dir == std::ios_base::end
- ? s.size() + off
- : this->off + off);
- }
- return pos_type(this->off);
- }
-
- pos_type seekpos(pos_type pos, std::ios_base::openmode which)
- {
- return seekoff(pos, std::ios_base::beg, which);
- }
-
- std::streamsize showmanyc()
- {
- return s.size() - off;
- }
-
- int_type underflow()
- {
- if (typename string_type::size_type(off) == s.size())
- return traits_type::eof();
- return traits_type::to_int_type(s[off]);
- }
-
- int_type uflow()
- {
- if (typename string_type::size_type(off) == s.size())
- return traits_type::eof();
- return traits_type::to_int_type(s[off++]);
- }
-
- int_type pbackfail(int_type ch)
- {
- if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
- return traits_type::eof();
-
- return traits_type::to_int_type(s[--off]);
- }
-
-};
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
-{
- typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
- buf_type buf;
-public:
- basic_istringstream_nocopy(const typename buf_type::string_type & s) :
- std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
-};
-
-typedef basic_istringstream_nocopy<char> istringstream_nocopy;
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 105fadb15..832aee783 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -1,5 +1,6 @@
#include "logging.hh"
#include "util.hh"
+#include "config.hh"
#include <atomic>
#include <nlohmann/json.hpp>
@@ -7,6 +8,10 @@
namespace nix {
+LoggerSettings loggerSettings;
+
+static GlobalConfig::Register r1(&loggerSettings);
+
static thread_local ActivityId curActivity = 0;
ActivityId getCurActivity()
@@ -72,7 +77,7 @@ public:
void logEI(const ErrorInfo & ei) override
{
std::stringstream oss;
- oss << ei;
+ showErrorInfo(oss, ei, loggerSettings.showTrace.get());
log(ei.level, oss.str());
}
@@ -173,7 +178,7 @@ struct JSONLogger : Logger {
void logEI(const ErrorInfo & ei) override
{
std::ostringstream oss;
- oss << ei;
+ showErrorInfo(oss, ei, loggerSettings.showTrace.get());
nlohmann::json json;
json["action"] = "msg";
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index b1583eced..09619aac6 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -2,6 +2,7 @@
#include "types.hh"
#include "error.hh"
+#include "config.hh"
namespace nix {
@@ -34,6 +35,16 @@ typedef enum {
typedef uint64_t ActivityId;
+struct LoggerSettings : Config
+{
+ Setting<bool> showTrace{this,
+ false,
+ "show-trace",
+ "Whether to show a stack trace on evaluation errors."};
+};
+
+extern LoggerSettings loggerSettings;
+
class Logger
{
friend struct Activity;
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index c8b71188f..00c945113 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -322,5 +322,18 @@ void StringSink::operator () (const unsigned char * data, size_t len)
s->append((const char *) data, len);
}
+size_t ChainSource::read(unsigned char * data, size_t len)
+{
+ if (useSecond) {
+ return source2.read(data, len);
+ } else {
+ try {
+ return source1.read(data, len);
+ } catch (EndOfFile &) {
+ useSecond = true;
+ return this->read(data, len);
+ }
+ }
+}
}
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index a04118512..c29c6b29b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -166,17 +166,30 @@ struct StringSource : Source
};
-/* Adapter class of a Source that saves all data read to `s'. */
+/* A sink that writes all incoming data to two other sinks. */
+struct TeeSink : Sink
+{
+ Sink & sink1, & sink2;
+ TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
+ virtual void operator () (const unsigned char * data, size_t len)
+ {
+ sink1(data, len);
+ sink2(data, len);
+ }
+};
+
+
+/* Adapter class of a Source that saves all data read to a sink. */
struct TeeSource : Source
{
Source & orig;
- ref<std::string> data;
- TeeSource(Source & orig)
- : orig(orig), data(make_ref<std::string>()) { }
+ Sink & sink;
+ TeeSource(Source & orig, Sink & sink)
+ : orig(orig), sink(sink) { }
size_t read(unsigned char * data, size_t len)
{
size_t n = orig.read(data, len);
- this->data->append((const char *) data, n);
+ sink(data, n);
return n;
}
};
@@ -243,6 +256,19 @@ struct LambdaSource : Source
}
};
+/* Chain two sources together so after the first is exhausted, the second is
+ used */
+struct ChainSource : Source
+{
+ Source & source1, & source2;
+ bool useSecond = false;
+ ChainSource(Source & s1, Source & s2)
+ : source1(s1), source2(s2)
+ { }
+
+ size_t read(unsigned char * data, size_t len) override;
+};
+
/* Convert a function that feeds data into a Sink into a Source. The
Source executes the function as a coroutine. */
@@ -286,14 +312,14 @@ T readNum(Source & source)
source(buf, sizeof(buf));
uint64_t n =
- ((unsigned long long) buf[0]) |
- ((unsigned long long) buf[1] << 8) |
- ((unsigned long long) buf[2] << 16) |
- ((unsigned long long) buf[3] << 24) |
- ((unsigned long long) buf[4] << 32) |
- ((unsigned long long) buf[5] << 40) |
- ((unsigned long long) buf[6] << 48) |
- ((unsigned long long) buf[7] << 56);
+ ((uint64_t) buf[0]) |
+ ((uint64_t) buf[1] << 8) |
+ ((uint64_t) buf[2] << 16) |
+ ((uint64_t) buf[3] << 24) |
+ ((uint64_t) buf[4] << 32) |
+ ((uint64_t) buf[5] << 40) |
+ ((uint64_t) buf[6] << 48) |
+ ((uint64_t) buf[7] << 56);
if (n > std::numeric_limits<T>::max())
throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
@@ -336,4 +362,27 @@ Source & operator >> (Source & in, bool & b)
}
+/* An adapter that converts a std::basic_istream into a source. */
+struct StreamToSourceAdapter : Source
+{
+ std::shared_ptr<std::basic_istream<char>> istream;
+
+ StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
+ : istream(istream)
+ { }
+
+ size_t read(unsigned char * data, size_t len) override
+ {
+ if (!istream->read((char *) data, len)) {
+ if (istream->eof()) {
+ if (istream->gcount() == 0)
+ throw EndOfFile("end of file");
+ } else
+ throw Error("I/O error in StreamToSourceAdapter");
+ }
+ return istream->gcount();
+ }
+};
+
+
}
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 6a6fb4ac3..ad588055f 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -11,6 +11,13 @@ namespace nix {
* logEI
* --------------------------------------------------------------------------*/
+ const char *test_file =
+ "previous line of code\n"
+ "this is the problem line of code\n"
+ "next line of code\n";
+ const char *one_liner =
+ "this is the other problem line of code";
+
TEST(logEI, catpuresBasicProperties) {
MakeError(TestError, Error);
@@ -137,7 +144,6 @@ namespace nix {
* logError
* --------------------------------------------------------------------------*/
-
TEST(logError, logErrorWithoutHintOrCode) {
testing::internal::CaptureStderr();
@@ -152,7 +158,7 @@ namespace nix {
TEST(logError, logErrorWithPreviousAndNextLinesOfCode) {
SymbolTable testTable;
- auto problem_file = testTable.create("myfile.nix");
+ auto problem_file = testTable.create(test_file);
testing::internal::CaptureStderr();
@@ -162,21 +168,16 @@ namespace nix {
.hint = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
- .nixCode = NixCode {
- .errPos = Pos(problem_file, 40, 13),
- .prevLineOfCode = "previous line of code",
- .errLineOfCode = "this is the problem line of code",
- .nextLineOfCode = "next line of code",
- }});
-
+ .errPos = Pos(foString, problem_file, 02, 13),
+ });
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror with code lines\n\n 39| previous line of code\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 41| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nerror with code lines\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
}
- TEST(logError, logErrorWithoutLinesOfCode) {
+ TEST(logError, logErrorWithInvalidFile) {
SymbolTable testTable;
- auto problem_file = testTable.create("myfile.nix");
+ auto problem_file = testTable.create("invalid filename");
testing::internal::CaptureStderr();
logError({
@@ -185,28 +186,23 @@ namespace nix {
.hint = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
- .nixCode = NixCode {
- .errPos = Pos(problem_file, 40, 13)
- }});
+ .errPos = Pos(foFile, problem_file, 02, 13)
+ });
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
}
TEST(logError, logErrorWithOnlyHintAndName) {
- SymbolTable testTable;
- auto problem_file = testTable.create("myfile.nix");
testing::internal::CaptureStderr();
logError({
.name = "error name",
.hint = hintfmt("hint %1%", "only"),
- .nixCode = NixCode {
- .errPos = Pos(problem_file, 40, 13)
- }});
+ });
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nhint \x1B[33;1monly\x1B[0m\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\nhint \x1B[33;1monly\x1B[0m\n");
}
@@ -219,18 +215,18 @@ namespace nix {
logWarning({
.name = "name",
- .description = "error description",
+ .description = "warning description",
.hint = hintfmt("there was a %1%", "warning"),
});
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nerror description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- name --- error-unit-test\x1B[0m\nwarning description\n\nthere was a \x1B[33;1mwarning\x1B[0m\n");
}
TEST(logWarning, logWarningWithFileLineNumAndCode) {
SymbolTable testTable;
- auto problem_file = testTable.create("myfile.nix");
+ auto problem_file = testTable.create(test_file);
testing::internal::CaptureStderr();
@@ -240,19 +236,74 @@ namespace nix {
.hint = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
- .nixCode = NixCode {
- .errPos = Pos(problem_file, 40, 13),
- .prevLineOfCode = std::nullopt,
- .errLineOfCode = "this is the problem line of code",
- .nextLineOfCode = std::nullopt
- }});
+ .errPos = Pos(foStdin, problem_file, 2, 13),
+ });
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\nin file: \x1B[34;1mmyfile.nix (40:13)\x1B[0m\n\nwarning description\n\n 40| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[33;1mwarning:\x1B[0m\x1B[34;1m --- warning name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from stdin\x1B[0m\n\nwarning description\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
}
/* ----------------------------------------------------------------------------
+ * traces
+ * --------------------------------------------------------------------------*/
+
+ TEST(addTrace, showTracesWithShowTrace) {
+ SymbolTable testTable;
+ auto problem_file = testTable.create(test_file);
+ auto oneliner_file = testTable.create(one_liner);
+ auto invalidfilename = testTable.create("invalid filename");
+
+ auto e = AssertionError(ErrorInfo {
+ .name = "wat",
+ .description = "show-traces",
+ .hint = hintfmt("it has been %1% days since our last error", "zero"),
+ .errPos = Pos(foString, problem_file, 2, 13),
+ });
+
+ e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
+ e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
+ e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
+
+ testing::internal::CaptureStderr();
+
+ loggerSettings.showTrace.assign(true);
+
+ logError(e.info());
+
+ auto str = testing::internal::GetCapturedStderr();
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
+ }
+
+ TEST(addTrace, hideTracesWithoutShowTrace) {
+ SymbolTable testTable;
+ auto problem_file = testTable.create(test_file);
+ auto oneliner_file = testTable.create(one_liner);
+ auto invalidfilename = testTable.create("invalid filename");
+
+ auto e = AssertionError(ErrorInfo {
+ .name = "wat",
+ .description = "hide traces",
+ .hint = hintfmt("it has been %1% days since our last error", "zero"),
+ .errPos = Pos(foString, problem_file, 2, 13),
+ });
+
+ e.addTrace(Pos(foStdin, oneliner_file, 1, 19), "while trying to compute %1%", 42);
+ e.addTrace(std::nullopt, "while doing something without a %1%", "pos");
+ e.addTrace(Pos(foFile, invalidfilename, 100, 1), "missing %s", "nix file");
+
+ testing::internal::CaptureStderr();
+
+ loggerSettings.showTrace.assign(false);
+
+ logError(e.info());
+
+ auto str = testing::internal::GetCapturedStderr();
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nhide traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n");
+ }
+
+
+ /* ----------------------------------------------------------------------------
* hintfmt
* --------------------------------------------------------------------------*/
@@ -289,4 +340,22 @@ namespace nix {
"what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
}
+
+ /* ----------------------------------------------------------------------------
+ * ErrPos
+ * --------------------------------------------------------------------------*/
+
+ TEST(errpos, invalidPos) {
+
+ // contains an invalid symbol, which we should not dereference!
+ Pos invalid;
+
+ // constructing without access violation.
+ ErrPos ep(invalid);
+
+ // assignment without access violation.
+ ep = invalid;
+
+ }
+
}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1268b146a..97d278581 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -23,6 +23,7 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/wait.h>
+#include <sys/time.h>
#include <sys/un.h>
#include <unistd.h>
@@ -79,7 +80,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
}
-Path absPath(Path path, std::optional<Path> dir)
+Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks)
{
if (path[0] != '/') {
if (!dir) {
@@ -100,7 +101,7 @@ Path absPath(Path path, std::optional<Path> dir)
}
path = *dir + "/" + path;
}
- return canonPath(path);
+ return canonPath(path, resolveSymlinks);
}
@@ -345,7 +346,6 @@ void writeFile(const Path & path, Source & source, mode_t mode)
}
}
-
string readLine(int fd)
{
string s;
@@ -374,7 +374,7 @@ void writeLine(int fd, string s)
}
-static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
{
checkInterrupt();
@@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by
}
}
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(const Path & path, uint64_t & bytesFreed)
{
Path dir = dirOf(path);
if (dir == "")
@@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
void deletePath(const Path & path)
{
- unsigned long long dummy;
+ uint64_t dummy;
deletePath(path, dummy);
}
-void deletePath(const Path & path, unsigned long long & bytesFreed)
+void deletePath(const Path & path, uint64_t & bytesFreed)
{
//Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
bytesFreed = 0;
@@ -581,20 +581,31 @@ Paths createDirs(const Path & path)
}
-void createSymlink(const Path & target, const Path & link)
+void createSymlink(const Path & target, const Path & link,
+ std::optional<time_t> mtime)
{
if (symlink(target.c_str(), link.c_str()))
throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+ if (mtime) {
+ struct timeval times[2];
+ times[0].tv_sec = *mtime;
+ times[0].tv_usec = 0;
+ times[1].tv_sec = *mtime;
+ times[1].tv_usec = 0;
+ if (lutimes(link.c_str(), times))
+ throw SysError("setting time of symlink '%s'", link);
+ }
}
-void replaceSymlink(const Path & target, const Path & link)
+void replaceSymlink(const Path & target, const Path & link,
+ std::optional<time_t> mtime)
{
for (unsigned int n = 0; true; n++) {
Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
try {
- createSymlink(target, tmp);
+ createSymlink(target, tmp, mtime);
} catch (SysError & e) {
if (e.errNo == EEXIST) continue;
throw;
@@ -1006,12 +1017,14 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
return res;
}
-
+// Output = "standard out" output stream
string runProgram(Path program, bool searchPath, const Strings & args,
const std::optional<std::string> & input)
{
RunOptions opts(program, args);
opts.searchPath = searchPath;
+ // This allows you to refer to a program with a pathname relative to the
+ // PATH variable.
opts.input = input;
auto res = runProgram(opts);
@@ -1022,6 +1035,7 @@ string runProgram(Path program, bool searchPath, const Strings & args,
return res.second;
}
+// Output = error code + "standard out" output stream
std::pair<int, std::string> runProgram(const RunOptions & options_)
{
RunOptions options(options_);
@@ -1094,6 +1108,8 @@ void runProgram2(const RunOptions & options)
if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
+ // This allows you to refer to a program with a pathname relative
+ // to the PATH variable.
else
execv(options.program.c_str(), stringsToCharPtrs(args_).data());
@@ -1565,7 +1581,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
struct sockaddr_un addr;
addr.sun_family = AF_UNIX;
- if (path.size() >= sizeof(addr.sun_path))
+ if (path.size() + 1 >= sizeof(addr.sun_path))
throw Error("socket path '%1%' is too long", path);
strcpy(addr.sun_path, path.c_str());
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3641daaec..6850b5a7a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -49,7 +49,9 @@ void clearEnv();
/* Return an absolutized path, resolving paths relative to the
specified directory, or the current directory otherwise. The path
is also canonicalised. */
-Path absPath(Path path, std::optional<Path> dir = {});
+Path absPath(Path path,
+ std::optional<Path> dir = {},
+ bool resolveSymlinks = false);
/* Canonicalise a path by removing all `.' or `..' components and
double or trailing slashes. Optionally resolves all symlink
@@ -123,7 +125,7 @@ void writeLine(int fd, string s);
second variant returns the number of bytes and blocks freed. */
void deletePath(const Path & path);
-void deletePath(const Path & path, unsigned long long & bytesFreed);
+void deletePath(const Path & path, uint64_t & bytesFreed);
std::string getUserName();
@@ -147,10 +149,12 @@ Path getDataDir();
Paths createDirs(const Path & path);
/* Create a symlink. */
-void createSymlink(const Path & target, const Path & link);
+void createSymlink(const Path & target, const Path & link,
+ std::optional<time_t> mtime = {});
/* Atomically create or replace a symlink. */
-void replaceSymlink(const Path & target, const Path & link);
+void replaceSymlink(const Path & target, const Path & link,
+ std::optional<time_t> mtime = {});
/* Wrappers arount read()/write() that read/write exactly the
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index f77de56ea..94412042f 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -174,7 +174,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--run-env") // obsolete
runEnv = true;
- else if (*arg == "--command" || *arg == "--run") {
+ else if (runEnv && (*arg == "--command" || *arg == "--run")) {
if (*arg == "--run")
interactive = false;
envCommand = getArg(*arg, arg, end) + "\nexit";
@@ -192,7 +192,7 @@ static void _main(int argc, char * * argv)
else if (*arg == "--pure") pure = true;
else if (*arg == "--impure") pure = false;
- else if (*arg == "--packages" || *arg == "-p")
+ else if (runEnv && (*arg == "--packages" || *arg == "-p"))
packages = true;
else if (inShebang && *arg == "-i") {
@@ -325,7 +325,7 @@ static void _main(int argc, char * * argv)
auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) {
/* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths,
willBuild, willSubstitute, unknown, downloadSize, narSize);
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index aa5ada3a6..bcf1d8518 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -67,10 +67,8 @@ static int _main(int argc, char * * argv)
deleteOlderThan = getArg(*arg, arg, end);
}
else if (*arg == "--dry-run") dryRun = true;
- else if (*arg == "--max-freed") {
- long long maxFreed = getIntArg<long long>(*arg, arg, end, true);
- options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
- }
+ else if (*arg == "--max-freed")
+ options.maxFreed = std::max(getIntArg<int64_t>(*arg, arg, end, true), (int64_t) 0);
else
return false;
return true;
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 8b0692035..ddd036070 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -381,7 +381,8 @@ static void queryInstSources(EvalState & state,
if (path.isDerivation()) {
elem.setDrvPath(state.store->printStorePath(path));
- elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out")));
+ auto outputs = state.store->queryDerivationOutputMap(path);
+ elem.setOutPath(state.store->printStorePath(outputs.at("out")));
if (name.size() >= drvExtension.size() &&
string(name, name.size() - drvExtension.size()) == drvExtension)
name = string(name, 0, name.size() - drvExtension.size());
@@ -593,7 +594,7 @@ static void upgradeDerivations(Globals & globals,
} else newElems.push_back(i);
} catch (Error & e) {
- e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName()));
+ e.addTrace(std::nullopt, "while trying to find an upgrade for '%s'", i.queryName());
throw;
}
}
@@ -1185,7 +1186,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
} catch (AssertionError & e) {
printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName());
} catch (Error & e) {
- e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName()));
+ e.addTrace(std::nullopt, "while querying the derivation named '%1%'", i.queryName());
throw;
}
}
@@ -1208,18 +1209,17 @@ static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs)
}
-static const int prevGen = -2;
+static constexpr GenerationNumber prevGen = std::numeric_limits<GenerationNumber>::max();
-static void switchGeneration(Globals & globals, int dstGen)
+static void switchGeneration(Globals & globals, GenerationNumber dstGen)
{
PathLocks lock;
lockProfile(lock, globals.profile);
- int curGen;
- Generations gens = findGenerations(globals.profile, curGen);
+ auto [gens, curGen] = findGenerations(globals.profile);
- Generation dst;
+ std::optional<Generation> dst;
for (auto & i : gens)
if ((dstGen == prevGen && i.number < curGen) ||
(dstGen >= 0 && i.number == dstGen))
@@ -1227,18 +1227,16 @@ static void switchGeneration(Globals & globals, int dstGen)
if (!dst) {
if (dstGen == prevGen)
- throw Error("no generation older than the current (%1%) exists",
- curGen);
+ throw Error("no generation older than the current (%1%) exists", curGen.value_or(0));
else
throw Error("generation %1% does not exist", dstGen);
}
- printInfo(format("switching from generation %1% to %2%")
- % curGen % dst.number);
+ printInfo("switching from generation %1% to %2%", curGen.value_or(0), dst->number);
if (globals.dryRun) return;
- switchLink(globals.profile, dst.path);
+ switchLink(globals.profile, dst->path);
}
@@ -1249,7 +1247,7 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg
if (opArgs.size() != 1)
throw UsageError("exactly one argument expected");
- int dstGen;
+ GenerationNumber dstGen;
if (!string2Int(opArgs.front(), dstGen))
throw UsageError("expected a generation number");
@@ -1278,8 +1276,7 @@ static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs
PathLocks lock;
lockProfile(lock, globals.profile);
- int curGen;
- Generations gens = findGenerations(globals.profile, curGen);
+ auto [gens, curGen] = findGenerations(globals.profile);
RunPager pager;
@@ -1308,14 +1305,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
if(opArgs.front().size() < 2)
throw Error("invalid number of generations ‘%1%’", opArgs.front());
string str_max = string(opArgs.front(), 1, opArgs.front().size());
- int max;
+ GenerationNumber max;
if (!string2Int(str_max, max) || max == 0)
throw Error("invalid number of generations to keep ‘%1%’", opArgs.front());
deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
} else {
- std::set<unsigned int> gens;
+ std::set<GenerationNumber> gens;
for (auto & i : opArgs) {
- unsigned int n;
+ GenerationNumber n;
if (!string2Int(i, n))
throw UsageError("invalid generation number '%1%'", i);
gens.insert(n);
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
index 40b05a2f3..65d8ec6b6 100644
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ b/src/nix-prefetch-url/nix-prefetch-url.cc
@@ -153,14 +153,15 @@ static int _main(int argc, char * * argv)
/* If an expected hash is given, the file may already exist in
the store. */
- Hash hash, expectedHash(ht);
+ std::optional<Hash> expectedHash;
+ Hash hash(ht);
std::optional<StorePath> storePath;
if (args.size() == 2) {
expectedHash = Hash(args[1], ht);
const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- storePath = store->makeFixedOutputPath(recursive, expectedHash, name);
+ storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
if (store->isValidPath(*storePath))
- hash = expectedHash;
+ hash = *expectedHash;
else
storePath.reset();
}
@@ -200,22 +201,12 @@ static int _main(int argc, char * * argv)
tmpFile = unpacked;
}
- /* FIXME: inefficient; addToStore() will also hash
- this. */
- hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile);
+ const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- if (expectedHash != Hash(ht) && expectedHash != hash)
- throw Error("hash mismatch for '%1%'", uri);
-
- const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
- /* Copy the file to the Nix store. FIXME: if RemoteStore
- implemented addToStoreFromDump() and downloadFile()
- supported a sink, we could stream the download directly
- into the Nix store. */
- storePath = store->addToStore(name, tmpFile, recursive, ht);
-
- assert(*storePath == store->makeFixedOutputPath(recursive, hash, name));
+ auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
+ storePath = info.path;
+ assert(info.ca);
+ hash = getContentAddressHash(*info.ca);
}
stopProgressBar();
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 7d81bf54f..7b26970ef 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -77,7 +77,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have an output named '%s'",
store2->printStorePath(path.path), j);
- auto outPath = store2->printStorePath(i->second.path);
+ auto outPath = store2->printStorePath(i->second.path(*store, drv.name));
if (store2) {
if (gcRoot == "")
printGCWarning();
@@ -130,7 +130,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
for (auto & i : opArgs)
paths.push_back(store->followLinksToStorePathWithOutputs(i));
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
@@ -174,10 +174,10 @@ static void opAdd(Strings opFlags, Strings opArgs)
store. */
static void opAddFixed(Strings opFlags, Strings opArgs)
{
- auto recursive = FileIngestionMethod::Flat;
+ auto method = FileIngestionMethod::Flat;
for (auto & i : opFlags)
- if (i == "--recursive") recursive = FileIngestionMethod::Recursive;
+ if (i == "--recursive") method = FileIngestionMethod::Recursive;
else throw UsageError("unknown flag '%1%'", i);
if (opArgs.empty())
@@ -187,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs)
opArgs.pop_front();
for (auto & i : opArgs)
- cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo)));
+ std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow(baseNameOf(i), i, method, hashAlgo).path));
}
@@ -219,7 +219,7 @@ static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput,
auto drv = store->derivationFromPath(storePath);
StorePathSet outputs;
for (auto & i : drv.outputs)
- outputs.insert(i.second.path);
+ outputs.insert(i.second.path(*store, drv.name));
return outputs;
}
else return {storePath};
@@ -313,7 +313,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
if (forceRealise) realisePath({i2});
Derivation drv = store->derivationFromPath(i2);
for (auto & j : drv.outputs)
- cout << fmt("%1%\n", store->printStorePath(j.second.path));
+ cout << fmt("%1%\n", store->printStorePath(j.second.path(*store, drv.name)));
}
break;
}
@@ -372,8 +372,8 @@ static void opQuery(Strings opFlags, Strings opArgs)
for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) {
auto info = store->queryPathInfo(j);
if (query == qHash) {
- assert(info->narHash.type == htSHA256);
- cout << fmt("%s\n", info->narHash.to_string(Base32, true));
+ assert(info->narHash && info->narHash->type == htSHA256);
+ cout << fmt("%s\n", info->narHash->to_string(Base32, true));
} else if (query == qSize)
cout << fmt("%d\n", info->narSize);
}
@@ -572,10 +572,8 @@ static void opGC(Strings opFlags, Strings opArgs)
if (*i == "--print-roots") printRoots = true;
else if (*i == "--print-live") options.action = GCOptions::gcReturnLive;
else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead;
- else if (*i == "--max-freed") {
- long long maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true);
- options.maxFreed = maxFreed >= 0 ? maxFreed : 0;
- }
+ else if (*i == "--max-freed")
+ options.maxFreed = std::max(getIntArg<int64_t>(*i, i, opFlags.end(), true), (int64_t) 0);
else throw UsageError("bad sub-operation '%1%' in GC", *i);
if (!opArgs.empty()) throw UsageError("no arguments expected");
@@ -671,7 +669,7 @@ static void opImport(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected");
FdSource source(STDIN_FILENO);
- auto paths = store->importPaths(source, nullptr, NoCheckSigs);
+ auto paths = store->importPaths(source, NoCheckSigs);
for (auto & i : paths)
cout << fmt("%s\n", store->printStorePath(i)) << std::flush;
@@ -725,7 +723,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
auto path = store->followLinksToStorePath(i);
printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path));
auto info = store->queryPathInfo(path);
- HashSink sink(*info->narHash.type);
+ HashSink sink(info->narHash->type);
store->narFromPath(path, sink);
auto current = sink.finish();
if (current.first != info->narHash) {
@@ -734,7 +732,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
.hint = hintfmt(
"path '%s' was modified! expected hash '%s', got '%s'",
store->printStorePath(path),
- info->narHash.to_string(Base32, true),
+ info->narHash->to_string(Base32, true),
current.first.to_string(Base32, true))
});
status = 1;
@@ -831,7 +829,7 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths)
if (!path.isDerivation())
paths2.push_back({path});
- unsigned long long downloadSize, narSize;
+ uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
store->queryMissing(paths2,
willBuild, willSubstitute, unknown, downloadSize, narSize);
@@ -864,7 +862,9 @@ static void opServe(Strings opFlags, Strings opArgs)
out << info->narSize // downloadSize
<< info->narSize;
if (GET_PROTOCOL_MINOR(clientVersion) >= 4)
- out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs;
+ out << (info->narHash ? info->narHash->to_string(Base32, true) : "")
+ << renderContentAddress(info->ca)
+ << info->sigs;
} catch (InvalidPath &) {
}
}
@@ -878,7 +878,7 @@ static void opServe(Strings opFlags, Strings opArgs)
case cmdImportPaths: {
if (!writeAllowed) throw Error("importing paths is not allowed");
- store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking?
+ store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking?
out << 1; // indicate success
break;
}
@@ -914,9 +914,9 @@ static void opServe(Strings opFlags, Strings opArgs)
if (!writeAllowed) throw Error("building paths is not allowed");
- auto drvPath = store->parseStorePath(readString(in)); // informational only
+ auto drvPath = store->parseStorePath(readString(in));
BasicDerivation drv;
- readDerivation(in, *store, drv);
+ readDerivation(in, *store, drv, Derivation::nameFromPath(drvPath));
getBuildSettings();
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index f9d6de16e..e183cb8b5 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,6 +9,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
std::optional<std::string> namePart;
+ FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
CmdAddToStore()
{
@@ -21,6 +22,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
.labels = {"name"},
.handler = {&namePart},
});
+
+ addFlag({
+ .longName = "flat",
+ .shortName = 0,
+ .description = "add flat file to the Nix store",
+ .handler = {&ingestionMethod, FileIngestionMethod::Flat},
+ });
}
std::string description() override
@@ -45,12 +53,19 @@ struct CmdAddToStore : MixDryRun, StoreCommand
auto narHash = hashString(htSHA256, *sink.s);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
+ Hash hash = narHash;
+ if (ingestionMethod == FileIngestionMethod::Flat) {
+ HashSink hsink(htSHA256);
+ readFile(path, hsink);
+ hash = hsink.finish().first;
+ }
+
+ ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
info.narHash = narHash;
info.narSize = sink.s->size();
info.ca = std::optional { FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = info.narHash,
+ .method = ingestionMethod,
+ .hash = hash,
} };
if (!dryRun) {
diff --git a/src/nix/app.cc b/src/nix/app.cc
new file mode 100644
index 000000000..3935297cf
--- /dev/null
+++ b/src/nix/app.cc
@@ -0,0 +1,46 @@
+#include "installables.hh"
+#include "store-api.hh"
+#include "eval-inline.hh"
+#include "eval-cache.hh"
+#include "names.hh"
+
+namespace nix {
+
+App Installable::toApp(EvalState & state)
+{
+ auto [cursor, attrPath] = getCursor(state, true);
+
+ auto type = cursor->getAttr("type")->getString();
+
+ if (type == "app") {
+ auto [program, context] = cursor->getAttr("program")->getStringWithContext();
+
+ if (!state.store->isInStore(program))
+ throw Error("app program '%s' is not in the Nix store", program);
+
+ std::vector<StorePathWithOutputs> context2;
+ for (auto & [path, name] : context)
+ context2.push_back({state.store->parseStorePath(path), {name}});
+
+ return App {
+ .context = std::move(context2),
+ .program = program,
+ };
+ }
+
+ else if (type == "derivation") {
+ auto drvPath = cursor->forceDerivation();
+ auto outPath = cursor->getAttr(state.sOutPath)->getString();
+ auto outputName = cursor->getAttr(state.sOutputName)->getString();
+ auto name = cursor->getAttr(state.sName)->getString();
+ return App {
+ .context = { { drvPath, {outputName} } },
+ .program = outPath + "/bin/" + DrvName(name).name,
+ };
+ }
+
+ else
+ throw Error("attribute '%s' has unsupported type '%s'", attrPath, type);
+}
+
+}
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 850e09ce8..613cc15eb 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -1,3 +1,4 @@
+#include "eval.hh"
#include "command.hh"
#include "common-args.hh"
#include "shared.hh"
@@ -8,6 +9,7 @@ using namespace nix;
struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
{
Path outLink = "result";
+ BuildMode buildMode = bmNormal;
CmdBuild()
{
@@ -17,6 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
.description = "path of the symlink to the build result",
.labels = {"path"},
.handler = {&outLink},
+ .completer = completePath
});
addFlag({
@@ -24,6 +27,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
.description = "do not create a symlink to the build result",
.handler = {&outLink, Path("")},
});
+
+ addFlag({
+ .longName = "rebuild",
+ .description = "rebuild an already built package and compare the result to the existing store paths",
+ .handler = {&buildMode, bmCheck},
+ });
}
std::string description() override
@@ -44,14 +53,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
},
Example{
"To make a profile point at GNU Hello:",
- "nix build --profile /tmp/profile nixpkgs.hello"
+ "nix build --profile /tmp/profile nixpkgs#hello"
},
};
}
void run(ref<Store> store) override
{
- auto buildables = build(store, dryRun ? DryRun : Build, installables);
+ auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode);
if (dryRun) return;
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
new file mode 100644
index 000000000..eb3339f5d
--- /dev/null
+++ b/src/nix/bundle.cc
@@ -0,0 +1,129 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "fs-accessor.hh"
+
+using namespace nix;
+
+struct CmdBundle : InstallableCommand
+{
+ std::string bundler = "github:matthewbauer/nix-bundle";
+ std::optional<Path> outLink;
+
+ CmdBundle()
+ {
+ addFlag({
+ .longName = "bundler",
+ .description = "use custom bundler",
+ .labels = {"flake-url"},
+ .handler = {&bundler},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeFlakeRef(getStore(), prefix);
+ }}
+ });
+
+ addFlag({
+ .longName = "out-link",
+ .shortName = 'o',
+ .description = "path of the symlink to the build result",
+ .labels = {"path"},
+ .handler = {&outLink},
+ .completer = completePath
+ });
+ }
+
+ std::string description() override
+ {
+ return "bundle an application so that it works outside of the Nix store";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To bundle Hello:",
+ "nix bundle hello"
+ },
+ };
+ }
+
+ Category category() override { return catSecondary; }
+
+ Strings getDefaultFlakeAttrPaths() override
+ {
+ Strings res{"defaultApp." + settings.thisSystem.get()};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths())
+ res.push_back(s);
+ return res;
+ }
+
+ Strings getDefaultFlakeAttrPathPrefixes() override
+ {
+ Strings res{"apps." + settings.thisSystem.get() + ".", "packages"};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes())
+ res.push_back(s);
+ return res;
+ }
+
+ void run(ref<Store> store) override
+ {
+ auto evalState = getEvalState();
+
+ auto app = installable->toApp(*evalState);
+ store->buildPaths(app.context);
+
+ auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
+ const flake::LockFlags lockFlags{ .writeLockFile = false };
+ auto bundler = InstallableFlake(
+ evalState, std::move(bundlerFlakeRef),
+ Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
+ Strings({"bundlers."}), lockFlags);
+
+ Value * arg = evalState->allocValue();
+ evalState->mkAttrs(*arg, 2);
+
+ PathSet context;
+ for (auto & i : app.context)
+ context.insert("=" + store->printStorePath(i.path));
+ mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context);
+
+ mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
+
+ arg->attrs->sort();
+
+ auto vRes = evalState->allocValue();
+ evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
+
+ if (!evalState->isDerivation(*vRes))
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ auto attr1 = vRes->attrs->find(evalState->sDrvPath);
+ if (!attr1)
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ PathSet context2;
+ StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2));
+
+ auto attr2 = vRes->attrs->find(evalState->sOutPath);
+ if (!attr2)
+ throw Error("the bundler '%s' does not produce a derivation", bundler.what());
+
+ StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
+
+ store->buildPaths({{drvPath}});
+
+ auto outPathS = store->printStorePath(outPath);
+
+ auto info = store->queryPathInfo(outPath);
+ if (!info->references.empty())
+ throw Error("'%s' has references; a bundler must not leave any references", outPathS);
+
+ if (!outLink)
+ outLink = baseNameOf(app.program);
+
+ store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink), true);
+ }
+};
+
+static auto r2 = registerCommand<CmdBundle>("bundle");
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index c82819af8..97306107c 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -25,7 +25,11 @@ struct CmdCatStore : StoreCommand, MixCat
{
CmdCatStore()
{
- expectArg("path", &path);
+ expectArgs({
+ .label = "path",
+ .handler = {&path},
+ .completer = completePath
+ });
}
std::string description() override
@@ -47,7 +51,11 @@ struct CmdCatNar : StoreCommand, MixCat
CmdCatNar()
{
- expectArg("nar", &narPath);
+ expectArgs({
+ .label = "nar",
+ .handler = {&narPath},
+ .completer = completePath
+ });
expectArg("path", &path);
}
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 3651a9e9c..af36dda89 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -63,7 +63,7 @@ void StorePathsCommand::run(ref<Store> store)
}
else {
- for (auto & p : toStorePaths(store, realiseMode, installables))
+ for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
storePaths.push_back(p);
if (recursive) {
@@ -80,7 +80,7 @@ void StorePathsCommand::run(ref<Store> store)
void StorePathCommand::run(ref<Store> store)
{
- auto storePaths = toStorePaths(store, NoBuild, installables);
+ auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
if (storePaths.size() != 1)
throw UsageError("this command requires exactly one store path");
@@ -108,6 +108,7 @@ MixProfile::MixProfile()
.description = "profile to update",
.labels = {"path"},
.handler = {&profile},
+ .completer = completePath
});
}
diff --git a/src/nix/command.hh b/src/nix/command.hh
index 959d5f19d..bc46a2028 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -4,12 +4,19 @@
#include "args.hh"
#include "common-eval-args.hh"
#include "path.hh"
-#include "eval.hh"
+#include "flake/lockfile.hh"
+#include "store-api.hh"
+
+#include <optional>
namespace nix {
extern std::string programPath;
+class EvalState;
+struct Pos;
+class Store;
+
static constexpr Command::Category catSecondary = 100;
static constexpr Command::Category catUtility = 101;
static constexpr Command::Category catNixInstallation = 102;
@@ -27,28 +34,64 @@ private:
std::shared_ptr<Store> _store;
};
-struct SourceExprCommand : virtual StoreCommand, MixEvalArgs
+struct EvalCommand : virtual StoreCommand, MixEvalArgs
{
- Path file;
+ ref<EvalState> getEvalState();
+
+ std::shared_ptr<EvalState> evalState;
+};
+
+struct MixFlakeOptions : virtual Args, EvalCommand
+{
+ flake::LockFlags lockFlags;
+
+ MixFlakeOptions();
+
+ virtual std::optional<FlakeRef> getFlakeRefForCompletion()
+ { return {}; }
+};
+
+/* How to handle derivations in commands that operate on store paths. */
+enum class OperateOn {
+ /* Operate on the output path. */
+ Output,
+ /* Operate on the .drv path. */
+ Derivation
+};
+
+struct SourceExprCommand : virtual Args, MixFlakeOptions
+{
+ std::optional<Path> file;
+ std::optional<std::string> expr;
+
+ // FIXME: move this; not all commands (e.g. 'nix run') use it.
+ OperateOn operateOn = OperateOn::Output;
SourceExprCommand();
- /* Return a value representing the Nix expression from which we
- are installing. This is either the file specified by ‘--file’,
- or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs
- = import ...; bla = import ...; }’. */
- Value * getSourceExpr(EvalState & state);
+ std::vector<std::shared_ptr<Installable>> parseInstallables(
+ ref<Store> store, std::vector<std::string> ss);
- ref<EvalState> getEvalState();
+ std::shared_ptr<Installable> parseInstallable(
+ ref<Store> store, const std::string & installable);
-private:
+ virtual Strings getDefaultFlakeAttrPaths();
- std::shared_ptr<EvalState> evalState;
+ virtual Strings getDefaultFlakeAttrPathPrefixes();
- RootValue vSourceExpr;
+ void completeInstallable(std::string_view prefix);
};
-enum RealiseMode { Build, NoBuild, DryRun };
+enum class Realise {
+ /* Build the derivation. Postcondition: the
+ derivation outputs exist. */
+ Outputs,
+ /* Don't build the derivation. Postcondition: the store derivation
+ exists. */
+ Derivation,
+ /* Evaluate in dry-run mode. Postcondition: nothing. */
+ Nothing
+};
/* A command that operates on a list of "installables", which can be
store paths, attribute paths, Nix expressions, etc. */
@@ -56,15 +99,14 @@ struct InstallablesCommand : virtual Args, SourceExprCommand
{
std::vector<std::shared_ptr<Installable>> installables;
- InstallablesCommand()
- {
- expectArgs("installables", &_installables);
- }
+ InstallablesCommand();
void prepare() override;
virtual bool useDefaultInstallables() { return true; }
+ std::optional<FlakeRef> getFlakeRefForCompletion() override;
+
private:
std::vector<std::string> _installables;
@@ -75,16 +117,18 @@ struct InstallableCommand : virtual Args, SourceExprCommand
{
std::shared_ptr<Installable> installable;
- InstallableCommand()
- {
- expectArg("installable", &_installable);
- }
+ InstallableCommand();
void prepare() override;
+ std::optional<FlakeRef> getFlakeRefForCompletion() override
+ {
+ return parseFlakeRef(_installable, absPath("."));
+ }
+
private:
- std::string _installable;
+ std::string _installable{"."};
};
/* A command that operates on zero or more store paths. */
@@ -97,7 +141,7 @@ private:
protected:
- RealiseMode realiseMode = NoBuild;
+ Realise realiseMode = Realise::Derivation;
public:
@@ -141,17 +185,15 @@ static RegisterCommand registerCommand(const std::string & name)
return RegisterCommand(name, [](){ return make_ref<T>(); });
}
-std::shared_ptr<Installable> parseInstallable(
- SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
- bool useDefaultInstallables);
-
-Buildables build(ref<Store> store, RealiseMode mode,
- std::vector<std::shared_ptr<Installable>> installables);
+Buildables build(ref<Store> store, Realise mode,
+ std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
-std::set<StorePath> toStorePaths(ref<Store> store, RealiseMode mode,
+std::set<StorePath> toStorePaths(ref<Store> store,
+ Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables);
-StorePath toStorePath(ref<Store> store, RealiseMode mode,
+StorePath toStorePath(ref<Store> store,
+ Realise mode, OperateOn operateOn,
std::shared_ptr<Installable> installable);
std::set<StorePath> toDerivations(ref<Store> store,
@@ -194,4 +236,19 @@ struct MixEnvironment : virtual Args {
void setEnviron();
};
+void completeFlakeRef(ref<Store> store, std::string_view prefix);
+
+void completeFlakeRefWithFragment(
+ ref<EvalState> evalState,
+ flake::LockFlags lockFlags,
+ Strings attrPathPrefixes,
+ const Strings & defaultFlakeAttrPaths,
+ std::string_view prefix);
+
+void printClosureDiff(
+ ref<Store> store,
+ const StorePath & beforePath,
+ const StorePath & afterPath,
+ std::string_view indent);
+
}
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 64099f476..811c8ab34 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -45,6 +45,8 @@ struct CmdCopy : StorePathsCommand
.description = "whether to try substitutes on the destination store (only supported by SSH)",
.handler = {&substitute, Substitute},
});
+
+ realiseMode = Realise::Outputs;
}
std::string description() override
@@ -70,11 +72,11 @@ struct CmdCopy : StorePathsCommand
#ifdef ENABLE_S3
Example{
"To copy Hello to an S3 binary cache:",
- "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello"
+ "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello"
},
Example{
"To copy Hello to an S3-compatible binary cache:",
- "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello"
+ "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello"
},
#endif
};
@@ -87,11 +89,16 @@ struct CmdCopy : StorePathsCommand
return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
}
- void run(ref<Store> srcStore, StorePaths storePaths) override
+ void run(ref<Store> store) override
{
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
+ StorePathsCommand::run(store);
+ }
+
+ void run(ref<Store> srcStore, StorePaths storePaths) override
+ {
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 037987313..a0c119e43 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -50,7 +50,7 @@ BuildEnvironment readEnvironment(const Path & path)
R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
static std::string indexedArrayRegex =
- R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
+ R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
static std::regex varRegex(
"^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
@@ -68,22 +68,22 @@ BuildEnvironment readEnvironment(const Path & path)
std::smatch match;
- if (std::regex_search(pos, file.cend(), match, declareRegex)) {
+ if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
exported.insert(match[1]);
}
- else if (std::regex_search(pos, file.cend(), match, varRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
}
- else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
pos = match[0].second;
res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
}
- else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
+ else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
res.bashFunctions = std::string(pos, file.cend());
break;
}
@@ -130,18 +130,16 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
drvName += "-env";
for (auto & output : drv.outputs)
drv.env.erase(output.first);
+ drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
drv.env["out"] = "";
+ drv.env["_outputs_saved"] = drv.env["outputs"];
drv.env["outputs"] = "out";
drv.inputSrcs.insert(std::move(getEnvShPath));
Hash h = hashDerivationModulo(*store, drv, true);
auto shellOutPath = store->makeOutputPath("out", h, drvName);
- drv.outputs.insert_or_assign("out", DerivationOutput {
- .path = shellOutPath,
- .hash = FixedOutputHash {
- .method = FileIngestionMethod::Flat,
- .hash = Hash { },
- },
- });
+ drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
+ .path = shellOutPath
+ } });
drv.env["out"] = store->printStorePath(shellOutPath);
auto shellDrvPath2 = writeDerivation(store, drv, drvName);
@@ -207,6 +205,11 @@ struct Common : InstallableCommand, MixProfile
out << "eval \"$shellHook\"\n";
}
+ Strings getDefaultFlakeAttrPaths() override
+ {
+ return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()};
+ }
+
StorePath getShellOutPath(ref<Store> store)
{
auto path = installable->getStorePath();
@@ -265,11 +268,15 @@ struct CmdDevelop : Common, MixEnvironment
return {
Example{
"To get the build environment of GNU hello:",
- "nix develop nixpkgs.hello"
+ "nix develop nixpkgs#hello"
+ },
+ Example{
+ "To get the build environment of the default package of flake in the current directory:",
+ "nix develop"
},
Example{
"To store the build environment in a profile:",
- "nix develop --profile /tmp/my-shell nixpkgs.hello"
+ "nix develop --profile /tmp/my-shell nixpkgs#hello"
},
Example{
"To use a build environment previously recorded in a profile:",
@@ -300,12 +307,28 @@ struct CmdDevelop : Common, MixEnvironment
stopProgressBar();
- auto shell = getEnv("SHELL").value_or("bash");
-
setEnviron();
// prevent garbage collection until shell exits
setenv("NIX_GCROOT", gcroot.data(), 1);
+ Path shell = "bash";
+
+ try {
+ auto state = getEvalState();
+
+ auto bashInstallable = std::make_shared<InstallableFlake>(
+ state,
+ std::move(installable->nixpkgsFlakeRef()),
+ Strings{"bashInteractive"},
+ Strings{"legacyPackages." + settings.thisSystem.get() + "."},
+ lockFlags);
+
+ shell = state->store->printStorePath(
+ toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";
+ } catch (Error &) {
+ ignoreException();
+ }
+
auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
restoreAffinity();
@@ -329,7 +352,7 @@ struct CmdPrintDevEnv : Common
return {
Example{
"To apply the build environment of GNU hello to the current shell:",
- ". <(nix print-dev-env nixpkgs.hello)"
+ ". <(nix print-dev-env nixpkgs#hello)"
},
};
}
diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc
new file mode 100644
index 000000000..4199dae0f
--- /dev/null
+++ b/src/nix/diff-closures.cc
@@ -0,0 +1,146 @@
+#include "command.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "common-args.hh"
+#include "names.hh"
+
+#include <regex>
+
+namespace nix {
+
+struct Info
+{
+ std::string outputName;
+};
+
+// name -> version -> store paths
+typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths;
+
+GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel)
+{
+ StorePathSet closure;
+ store->computeFSClosure({toplevel}, closure);
+
+ GroupedPaths groupedPaths;
+
+ for (auto & path : closure) {
+ /* Strip the output name. Unfortunately this is ambiguous (we
+ can't distinguish between output names like "bin" and
+ version suffixes like "unstable"). */
+ static std::regex regex("(.*)-([a-z]+|lib32|lib64)");
+ std::smatch match;
+ std::string name(path.name());
+ std::string outputName;
+ if (std::regex_match(name, match, regex)) {
+ name = match[1];
+ outputName = match[2];
+ }
+
+ DrvName drvName(name);
+ groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName });
+ }
+
+ return groupedPaths;
+}
+
+std::string showVersions(const std::set<std::string> & versions)
+{
+ if (versions.empty()) return "∅";
+ std::set<std::string> versions2;
+ for (auto & version : versions)
+ versions2.insert(version.empty() ? "ε" : version);
+ return concatStringsSep(", ", versions2);
+}
+
+void printClosureDiff(
+ ref<Store> store,
+ const StorePath & beforePath,
+ const StorePath & afterPath,
+ std::string_view indent)
+{
+ auto beforeClosure = getClosureInfo(store, beforePath);
+ auto afterClosure = getClosureInfo(store, afterPath);
+
+ std::set<std::string> allNames;
+ for (auto & [name, _] : beforeClosure) allNames.insert(name);
+ for (auto & [name, _] : afterClosure) allNames.insert(name);
+
+ for (auto & name : allNames) {
+ auto & beforeVersions = beforeClosure[name];
+ auto & afterVersions = afterClosure[name];
+
+ auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions)
+ {
+ uint64_t sum = 0;
+ for (auto & [_, paths] : versions)
+ for (auto & [path, _] : paths)
+ sum += store->queryPathInfo(path)->narSize;
+ return sum;
+ };
+
+ auto beforeSize = totalSize(beforeVersions);
+ auto afterSize = totalSize(afterVersions);
+ auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize;
+ auto showDelta = abs(sizeDelta) >= 8 * 1024;
+
+ std::set<std::string> removed, unchanged;
+ for (auto & [version, _] : beforeVersions)
+ if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version);
+
+ std::set<std::string> added;
+ for (auto & [version, _] : afterVersions)
+ if (!beforeVersions.count(version)) added.insert(version);
+
+ if (showDelta || !removed.empty() || !added.empty()) {
+ std::vector<std::string> items;
+ if (!removed.empty() || !added.empty())
+ items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added)));
+ if (showDelta)
+ items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0));
+ std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items));
+ }
+ }
+}
+
+}
+
+using namespace nix;
+
+struct CmdDiffClosures : SourceExprCommand
+{
+ std::string _before, _after;
+
+ CmdDiffClosures()
+ {
+ expectArg("before", &_before);
+ expectArg("after", &_after);
+ }
+
+ std::string description() override
+ {
+ return "show what packages and versions were added and removed between two closures";
+ }
+
+ Category category() override { return catSecondary; }
+
+ Examples examples() override
+ {
+ return {
+ {
+ "To show what got added and removed between two versions of the NixOS system profile:",
+ "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
+ },
+ };
+ }
+
+ void run(ref<Store> store) override
+ {
+ auto before = parseInstallable(store, _before);
+ auto beforePath = toStorePath(store, Realise::Outputs, operateOn, before);
+ auto after = parseInstallable(store, _after);
+ auto afterPath = toStorePath(store, Realise::Outputs, operateOn, after);
+ printClosureDiff(store, beforePath, afterPath, "");
+ }
+};
+
+static auto r1 = registerCommand<CmdDiffClosures>("diff-closures");
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 067d3a973..378a3739c 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -20,7 +20,7 @@ struct CmdEdit : InstallableCommand
return {
Example{
"To open the Nix expression of the GNU Hello package:",
- "nix edit nixpkgs.hello"
+ "nix edit nixpkgs#hello"
},
};
}
@@ -45,6 +45,7 @@ struct CmdEdit : InstallableCommand
auto args = editorFor(pos);
+ restoreSignals();
execvp(args.front().c_str(), stringsToCharPtrs(args).data());
std::string command;
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 26e98ac2a..a8ca446be 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -12,10 +12,18 @@ using namespace nix;
struct CmdEval : MixJSON, InstallableCommand
{
bool raw = false;
+ std::optional<std::string> apply;
CmdEval()
{
mkFlag(0, "raw", "print strings unquoted", &raw);
+
+ addFlag({
+ .longName = "apply",
+ .description = "apply a function to each argument",
+ .labels = {"expr"},
+ .handler = {&apply},
+ });
}
std::string description() override
@@ -26,21 +34,25 @@ struct CmdEval : MixJSON, InstallableCommand
Examples examples() override
{
return {
- Example{
+ {
"To evaluate a Nix expression given on the command line:",
- "nix eval '(1 + 2)'"
+ "nix eval --expr '1 + 2'"
},
- Example{
+ {
"To evaluate a Nix expression from a file or URI:",
- "nix eval -f channel:nixos-17.09 hello.name"
+ "nix eval -f ./my-nixpkgs hello.name"
},
- Example{
+ {
"To get the current version of Nixpkgs:",
- "nix eval --raw nixpkgs.lib.version"
+ "nix eval --raw nixpkgs#lib.version"
},
- Example{
+ {
"To print the store path of the Hello package:",
- "nix eval --raw nixpkgs.hello"
+ "nix eval --raw nixpkgs#hello"
+ },
+ {
+ "To get a list of checks in the 'nix' flake:",
+ "nix eval nix#checks.x86_64-linux --apply builtins.attrNames"
},
};
}
@@ -57,6 +69,14 @@ struct CmdEval : MixJSON, InstallableCommand
auto v = installable->toValue(*state).first;
PathSet context;
+ if (apply) {
+ auto vApply = state->allocValue();
+ state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply);
+ auto vRes = state->allocValue();
+ state->callFunction(*vApply, *v, *vRes, noPos);
+ v = vRes;
+ }
+
if (raw) {
stopProgressBar();
std::cout << state->coerceToString(noPos, *v, context);
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
new file mode 100644
index 000000000..80d8654bc
--- /dev/null
+++ b/src/nix/flake.cc
@@ -0,0 +1,980 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "eval.hh"
+#include "eval-inline.hh"
+#include "flake/flake.hh"
+#include "get-drvs.hh"
+#include "store-api.hh"
+#include "derivations.hh"
+#include "attr-path.hh"
+#include "fetchers.hh"
+#include "registry.hh"
+#include "json.hh"
+#include "eval-cache.hh"
+
+#include <nlohmann/json.hpp>
+#include <queue>
+#include <iomanip>
+
+using namespace nix;
+using namespace nix::flake;
+
+class FlakeCommand : virtual Args, public MixFlakeOptions
+{
+ std::string flakeUrl = ".";
+
+public:
+
+ FlakeCommand()
+ {
+ expectArgs({
+ .label = "flake-url",
+ .optional = true,
+ .handler = {&flakeUrl},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeFlakeRef(getStore(), prefix);
+ }}
+ });
+ }
+
+ FlakeRef getFlakeRef()
+ {
+ return parseFlakeRef(flakeUrl, absPath(".")); //FIXME
+ }
+
+ Flake getFlake()
+ {
+ auto evalState = getEvalState();
+ return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries);
+ }
+
+ LockedFlake lockFlake()
+ {
+ return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags);
+ }
+
+ std::optional<FlakeRef> getFlakeRefForCompletion() override
+ {
+ return getFlakeRef();
+ }
+};
+
+static void printFlakeInfo(const Store & store, const Flake & flake)
+{
+ logger->stdout("Resolved URL: %s", flake.resolvedRef.to_string());
+ logger->stdout("Locked URL: %s", flake.lockedRef.to_string());
+ if (flake.description)
+ logger->stdout("Description: %s", *flake.description);
+ logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath));
+ if (auto rev = flake.lockedRef.input.getRev())
+ logger->stdout("Revision: %s", rev->to_string(Base16, false));
+ if (auto revCount = flake.lockedRef.input.getRevCount())
+ logger->stdout("Revisions: %s", *revCount);
+ if (auto lastModified = flake.lockedRef.input.getLastModified())
+ logger->stdout("Last modified: %s",
+ std::put_time(std::localtime(&*lastModified), "%F %T"));
+}
+
+static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
+{
+ nlohmann::json j;
+ if (flake.description)
+ j["description"] = *flake.description;
+ j["originalUrl"] = flake.originalRef.to_string();
+ j["original"] = attrsToJson(flake.originalRef.toAttrs());
+ j["resolvedUrl"] = flake.resolvedRef.to_string();
+ j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs());
+ j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
+ j["locked"] = attrsToJson(flake.lockedRef.toAttrs());
+ if (auto rev = flake.lockedRef.input.getRev())
+ j["revision"] = rev->to_string(Base16, false);
+ if (auto revCount = flake.lockedRef.input.getRevCount())
+ j["revCount"] = *revCount;
+ if (auto lastModified = flake.lockedRef.input.getLastModified())
+ j["lastModified"] = *lastModified;
+ j["path"] = store.printStorePath(flake.sourceInfo->storePath);
+ return j;
+}
+
+struct CmdFlakeUpdate : FlakeCommand
+{
+ std::string description() override
+ {
+ return "update flake lock file";
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ /* Use --refresh by default for 'nix flake update'. */
+ settings.tarballTtl = 0;
+
+ lockFlake();
+ }
+};
+
+static void enumerateOutputs(EvalState & state, Value & vFlake,
+ std::function<void(const std::string & name, Value & vProvide, const Pos & pos)> callback)
+{
+ state.forceAttrs(vFlake);
+
+ auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs"));
+ assert(aOutputs);
+
+ state.forceAttrs(*aOutputs->value);
+
+ for (auto & attr : *aOutputs->value->attrs)
+ callback(attr.name, *attr.value, *attr.pos);
+}
+
+struct CmdFlakeInfo : FlakeCommand, MixJSON
+{
+ std::string description() override
+ {
+ return "list info about a given flake";
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ auto flake = getFlake();
+
+ if (json) {
+ auto json = flakeToJson(*store, flake);
+ logger->stdout("%s", json.dump());
+ } else
+ printFlakeInfo(*store, flake);
+ }
+};
+
+struct CmdFlakeListInputs : FlakeCommand, MixJSON
+{
+ std::string description() override
+ {
+ return "list flake inputs";
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ auto flake = lockFlake();
+
+ if (json)
+ logger->stdout("%s", flake.lockFile.toJson());
+ else {
+ logger->stdout("%s", flake.flake.lockedRef);
+
+ std::unordered_set<std::shared_ptr<Node>> visited;
+
+ std::function<void(const Node & node, const std::string & prefix)> recurse;
+
+ recurse = [&](const Node & node, const std::string & prefix)
+ {
+ for (const auto & [i, input] : enumerate(node.inputs)) {
+ bool last = i + 1 == node.inputs.size();
+
+ if (auto lockedNode = std::get_if<0>(&input.second)) {
+ logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
+ prefix + (last ? treeLast : treeConn), input.first,
+ *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef);
+
+ bool firstVisit = visited.insert(*lockedNode).second;
+
+ if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine));
+ } else if (auto follows = std::get_if<1>(&input.second)) {
+ logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'",
+ prefix + (last ? treeLast : treeConn), input.first,
+ printInputPath(*follows));
+ }
+ }
+ };
+
+ visited.insert(flake.lockFile.root);
+ recurse(*flake.lockFile.root, "");
+ }
+ }
+};
+
+struct CmdFlakeCheck : FlakeCommand
+{
+ bool build = true;
+
+ CmdFlakeCheck()
+ {
+ addFlag({
+ .longName = "no-build",
+ .description = "do not build checks",
+ .handler = {&build, false}
+ });
+ }
+
+ std::string description() override
+ {
+ return "check whether the flake evaluates and run its tests";
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ settings.readOnlyMode = !build;
+
+ auto state = getEvalState();
+ auto flake = lockFlake();
+
+ // FIXME: rewrite to use EvalCache.
+
+ auto checkSystemName = [&](const std::string & system, const Pos & pos) {
+ // FIXME: what's the format of "system"?
+ if (system.find('-') == std::string::npos)
+ throw Error("'%s' is not a valid system type, at %s", system, pos);
+ };
+
+ auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ auto drvInfo = getDerivation(*state, v, false);
+ if (!drvInfo)
+ throw Error("flake attribute '%s' is not a derivation", attrPath);
+ // FIXME: check meta attributes
+ return store->parseStorePath(drvInfo->queryDrvPath());
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath));
+ throw;
+ }
+ };
+
+ std::vector<StorePathWithOutputs> drvPaths;
+
+ auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ #if 0
+ // FIXME
+ auto app = App(*state, v);
+ for (auto & i : app.context) {
+ auto [drvPathS, outputName] = decodeContext(i);
+ store->parseStorePath(drvPathS);
+ }
+ #endif
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath));
+ throw;
+ }
+ };
+
+ auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ state->forceValue(v, pos);
+ if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
+ throw Error("overlay does not take an argument named 'final'");
+ auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
+ if (!body || body->matchAttrs || std::string(body->arg) != "prev")
+ throw Error("overlay does not take an argument named 'prev'");
+ // FIXME: if we have a 'nixpkgs' input, use it to
+ // evaluate the overlay.
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath));
+ throw;
+ }
+ };
+
+ auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ state->forceValue(v, pos);
+ if (v.type == tLambda) {
+ if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
+ throw Error("module must match an open attribute set ('{ config, ... }')");
+ } else if (v.type == tAttrs) {
+ for (auto & attr : *v.attrs)
+ try {
+ state->forceValue(*attr.value, *attr.pos);
+ } catch (Error & e) {
+ e.addTrace(*attr.pos, hintfmt("while evaluating the option '%s'", attr.name));
+ throw;
+ }
+ } else
+ throw Error("module must be a function or an attribute set");
+ // FIXME: if we have a 'nixpkgs' input, use it to
+ // check the module.
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath));
+ throw;
+ }
+ };
+
+ std::function<void(const std::string & attrPath, Value & v, const Pos & pos)> checkHydraJobs;
+
+ checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ state->forceAttrs(v, pos);
+
+ if (state->isDerivation(v))
+ throw Error("jobset should not be a derivation at top-level");
+
+ for (auto & attr : *v.attrs) {
+ state->forceAttrs(*attr.value, *attr.pos);
+ if (!state->isDerivation(*attr.value))
+ checkHydraJobs(attrPath + "." + (std::string) attr.name,
+ *attr.value, *attr.pos);
+ }
+
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath));
+ throw;
+ }
+ };
+
+ auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ Activity act(*logger, lvlChatty, actUnknown,
+ fmt("checking NixOS configuration '%s'", attrPath));
+ Bindings & bindings(*state->allocBindings(0));
+ auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first;
+ state->forceAttrs(*vToplevel, pos);
+ if (!state->isDerivation(*vToplevel))
+ throw Error("attribute 'config.system.build.toplevel' is not a derivation");
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath));
+ throw;
+ }
+ };
+
+ auto checkTemplate = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ Activity act(*logger, lvlChatty, actUnknown,
+ fmt("checking template '%s'", attrPath));
+
+ state->forceAttrs(v, pos);
+
+ if (auto attr = v.attrs->get(state->symbols.create("path"))) {
+ if (attr->name == state->symbols.create("path")) {
+ PathSet context;
+ auto path = state->coerceToPath(*attr->pos, *attr->value, context);
+ if (!store->isInStore(path))
+ throw Error("template '%s' has a bad 'path' attribute");
+ // TODO: recursively check the flake in 'path'.
+ }
+ } else
+ throw Error("template '%s' lacks attribute 'path'", attrPath);
+
+ if (auto attr = v.attrs->get(state->symbols.create("description")))
+ state->forceStringNoCtx(*attr->value, *attr->pos);
+ else
+ throw Error("template '%s' lacks attribute 'description'", attrPath);
+
+ for (auto & attr : *v.attrs) {
+ std::string name(attr.name);
+ if (name != "path" && name != "description")
+ throw Error("template '%s' has unsupported attribute '%s'", attrPath, name);
+ }
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath));
+ throw;
+ }
+ };
+
+ auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) {
+ try {
+ state->forceValue(v, pos);
+ if (v.type != tLambda)
+ throw Error("bundler must be a function");
+ if (!v.lambda.fun->formals ||
+ v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() ||
+ v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end())
+ throw Error("bundler must take formal arguments 'program' and 'system'");
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath));
+ throw;
+ }
+ };
+
+ {
+ Activity act(*logger, lvlInfo, actUnknown, "evaluating flake");
+
+ auto vFlake = state->allocValue();
+ flake::callFlake(*state, flake, *vFlake);
+
+ enumerateOutputs(*state,
+ *vFlake,
+ [&](const std::string & name, Value & vOutput, const Pos & pos) {
+ Activity act(*logger, lvlChatty, actUnknown,
+ fmt("checking flake output '%s'", name));
+
+ try {
+ state->forceValue(vOutput, pos);
+
+ if (name == "checks") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs) {
+ checkSystemName(attr.name, *attr.pos);
+ state->forceAttrs(*attr.value, *attr.pos);
+ for (auto & attr2 : *attr.value->attrs) {
+ auto drvPath = checkDerivation(
+ fmt("%s.%s.%s", name, attr.name, attr2.name),
+ *attr2.value, *attr2.pos);
+ if ((std::string) attr.name == settings.thisSystem.get())
+ drvPaths.push_back({drvPath});
+ }
+ }
+ }
+
+ else if (name == "packages") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs) {
+ checkSystemName(attr.name, *attr.pos);
+ state->forceAttrs(*attr.value, *attr.pos);
+ for (auto & attr2 : *attr.value->attrs)
+ checkDerivation(
+ fmt("%s.%s.%s", name, attr.name, attr2.name),
+ *attr2.value, *attr2.pos);
+ }
+ }
+
+ else if (name == "apps") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs) {
+ checkSystemName(attr.name, *attr.pos);
+ state->forceAttrs(*attr.value, *attr.pos);
+ for (auto & attr2 : *attr.value->attrs)
+ checkApp(
+ fmt("%s.%s.%s", name, attr.name, attr2.name),
+ *attr2.value, *attr2.pos);
+ }
+ }
+
+ else if (name == "defaultPackage" || name == "devShell") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs) {
+ checkSystemName(attr.name, *attr.pos);
+ checkDerivation(
+ fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+ }
+
+ else if (name == "defaultApp") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs) {
+ checkSystemName(attr.name, *attr.pos);
+ checkApp(
+ fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+ }
+
+ else if (name == "legacyPackages") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs) {
+ checkSystemName(attr.name, *attr.pos);
+ // FIXME: do getDerivations?
+ }
+ }
+
+ else if (name == "overlay")
+ checkOverlay(name, vOutput, pos);
+
+ else if (name == "overlays") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkOverlay(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
+ else if (name == "nixosModule")
+ checkModule(name, vOutput, pos);
+
+ else if (name == "nixosModules") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkModule(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
+ else if (name == "nixosConfigurations") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkNixOSConfiguration(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
+ else if (name == "hydraJobs")
+ checkHydraJobs(name, vOutput, pos);
+
+ else if (name == "defaultTemplate")
+ checkTemplate(name, vOutput, pos);
+
+ else if (name == "templates") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkTemplate(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
+ else if (name == "defaultBundler")
+ checkBundler(name, vOutput, pos);
+
+ else if (name == "bundlers") {
+ state->forceAttrs(vOutput, pos);
+ for (auto & attr : *vOutput.attrs)
+ checkBundler(fmt("%s.%s", name, attr.name),
+ *attr.value, *attr.pos);
+ }
+
+ else
+ warn("unknown flake output '%s'", name);
+
+ } catch (Error & e) {
+ e.addTrace(pos, hintfmt("while checking flake output '%s'", name));
+ throw;
+ }
+ });
+ }
+
+ if (build && !drvPaths.empty()) {
+ Activity act(*logger, lvlInfo, actUnknown, "running flake checks");
+ store->buildPaths(drvPaths);
+ }
+ }
+};
+
+struct CmdFlakeInitCommon : virtual Args, EvalCommand
+{
+ std::string templateUrl = "templates";
+ Path destDir;
+
+ const Strings attrsPathPrefixes{"templates."};
+ const LockFlags lockFlags{ .writeLockFile = false };
+
+ CmdFlakeInitCommon()
+ {
+ addFlag({
+ .longName = "template",
+ .shortName = 't',
+ .description = "the template to use",
+ .labels = {"template"},
+ .handler = {&templateUrl},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeFlakeRefWithFragment(
+ getEvalState(),
+ lockFlags,
+ attrsPathPrefixes,
+ {"defaultTemplate"},
+ prefix);
+ }}
+ });
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ auto flakeDir = absPath(destDir);
+
+ auto evalState = getEvalState();
+
+ auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
+
+ auto installable = InstallableFlake(
+ evalState, std::move(templateFlakeRef),
+ Strings{templateName == "" ? "defaultTemplate" : templateName},
+ Strings(attrsPathPrefixes), lockFlags);
+
+ auto [cursor, attrPath] = installable.getCursor(*evalState, true);
+
+ auto templateDir = cursor->getAttr("path")->getString();
+
+ assert(store->isInStore(templateDir));
+
+ std::vector<Path> files;
+
+ std::function<void(const Path & from, const Path & to)> copyDir;
+ copyDir = [&](const Path & from, const Path & to)
+ {
+ createDirs(to);
+
+ for (auto & entry : readDirectory(from)) {
+ auto from2 = from + "/" + entry.name;
+ auto to2 = to + "/" + entry.name;
+ auto st = lstat(from2);
+ if (S_ISDIR(st.st_mode))
+ copyDir(from2, to2);
+ else if (S_ISREG(st.st_mode)) {
+ auto contents = readFile(from2);
+ if (pathExists(to2)) {
+ auto contents2 = readFile(to2);
+ if (contents != contents2)
+ throw Error("refusing to overwrite existing file '%s'", to2);
+ } else
+ writeFile(to2, contents);
+ }
+ else if (S_ISLNK(st.st_mode)) {
+ auto target = readLink(from2);
+ if (pathExists(to2)) {
+ if (readLink(to2) != target)
+ throw Error("refusing to overwrite existing symlink '%s'", to2);
+ } else
+ createSymlink(target, to2);
+ }
+ else
+ throw Error("file '%s' has unsupported type", from2);
+ files.push_back(to2);
+ }
+ };
+
+ copyDir(templateDir, flakeDir);
+
+ if (pathExists(flakeDir + "/.git")) {
+ Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" };
+ for (auto & s : files) args.push_back(s);
+ runProgram("git", true, args);
+ }
+ }
+};
+
+struct CmdFlakeInit : CmdFlakeInitCommon
+{
+ std::string description() override
+ {
+ return "create a flake in the current directory from a template";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To create a flake using the default template:",
+ "nix flake init"
+ },
+ Example{
+ "To see available templates:",
+ "nix flake show templates"
+ },
+ Example{
+ "To create a flake from a specific template:",
+ "nix flake init -t templates#nixos-container"
+ },
+ };
+ }
+
+ CmdFlakeInit()
+ {
+ destDir = ".";
+ }
+};
+
+struct CmdFlakeNew : CmdFlakeInitCommon
+{
+ std::string description() override
+ {
+ return "create a flake in the specified directory from a template";
+ }
+
+ CmdFlakeNew()
+ {
+ expectArgs({
+ .label = "dest-dir",
+ .handler = {&destDir},
+ .completer = completePath
+ });
+ }
+};
+
+struct CmdFlakeClone : FlakeCommand
+{
+ Path destDir;
+
+ std::string description() override
+ {
+ return "clone flake repository";
+ }
+
+ CmdFlakeClone()
+ {
+ addFlag({
+ .longName = "dest",
+ .shortName = 'f',
+ .description = "destination path",
+ .labels = {"path"},
+ .handler = {&destDir}
+ });
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ if (destDir.empty())
+ throw Error("missing flag '--dest'");
+
+ getFlakeRef().resolve(store).input.clone(destDir);
+ }
+};
+
+struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
+{
+ std::string dstUri;
+
+ CmdFlakeArchive()
+ {
+ addFlag({
+ .longName = "to",
+ .description = "URI of the destination Nix store",
+ .labels = {"store-uri"},
+ .handler = {&dstUri}
+ });
+ }
+
+ std::string description() override
+ {
+ return "copy a flake and all its inputs to a store";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To copy the dwarffs flake and its dependencies to a binary cache:",
+ "nix flake archive --to file:///tmp/my-cache dwarffs"
+ },
+ Example{
+ "To fetch the dwarffs flake and its dependencies to the local Nix store:",
+ "nix flake archive dwarffs"
+ },
+ Example{
+ "To print the store paths of the flake sources of NixOps without fetching them:",
+ "nix flake archive --json --dry-run nixops"
+ },
+ };
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ auto flake = lockFlake();
+
+ auto jsonRoot = json ? std::optional<JSONObject>(std::cout) : std::nullopt;
+
+ StorePathSet sources;
+
+ sources.insert(flake.flake.sourceInfo->storePath);
+ if (jsonRoot)
+ jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath));
+
+ // FIXME: use graph output, handle cycles.
+ std::function<void(const Node & node, std::optional<JSONObject> & jsonObj)> traverse;
+ traverse = [&](const Node & node, std::optional<JSONObject> & jsonObj)
+ {
+ auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional<JSONObject>();
+ for (auto & [inputName, input] : node.inputs) {
+ if (auto inputNode = std::get_if<0>(&input)) {
+ auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional<JSONObject>();
+ auto storePath =
+ dryRun
+ ? (*inputNode)->lockedRef.input.computeStorePath(*store)
+ : (*inputNode)->lockedRef.input.fetch(store).first.storePath;
+ if (jsonObj3)
+ jsonObj3->attr("path", store->printStorePath(storePath));
+ sources.insert(std::move(storePath));
+ traverse(**inputNode, jsonObj3);
+ }
+ }
+ };
+
+ traverse(*flake.lockFile.root, jsonRoot);
+
+ if (!dryRun && !dstUri.empty()) {
+ ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
+ copyPaths(store, dstStore, sources);
+ }
+ }
+};
+
+struct CmdFlakeShow : FlakeCommand
+{
+ bool showLegacy = false;
+ bool useEvalCache = true;
+
+ CmdFlakeShow()
+ {
+ addFlag({
+ .longName = "legacy",
+ .description = "show the contents of the 'legacyPackages' output",
+ .handler = {&showLegacy, true}
+ });
+
+ addFlag({
+ .longName = "no-eval-cache",
+ .description = "do not use the flake evaluation cache",
+ .handler = {[&]() { useEvalCache = false; }}
+ });
+ }
+
+ std::string description() override
+ {
+ return "show the outputs provided by a flake";
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ auto state = getEvalState();
+ auto flake = std::make_shared<LockedFlake>(lockFlake());
+
+ std::function<void(eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)> visit;
+
+ visit = [&](eval_cache::AttrCursor & visitor, const std::vector<Symbol> & attrPath, const std::string & headerPrefix, const std::string & nextPrefix)
+ {
+ Activity act(*logger, lvlInfo, actUnknown,
+ fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
+ try {
+ auto recurse = [&]()
+ {
+ logger->stdout("%s", headerPrefix);
+ auto attrs = visitor.getAttrs();
+ for (const auto & [i, attr] : enumerate(attrs)) {
+ bool last = i + 1 == attrs.size();
+ auto visitor2 = visitor.getAttr(attr);
+ auto attrPath2(attrPath);
+ attrPath2.push_back(attr);
+ visit(*visitor2, attrPath2,
+ fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr),
+ nextPrefix + (last ? treeNull : treeLine));
+ }
+ };
+
+ auto showDerivation = [&]()
+ {
+ auto name = visitor.getAttr(state->sName)->getString();
+
+ /*
+ std::string description;
+
+ if (auto aMeta = visitor.maybeGetAttr("meta")) {
+ if (auto aDescription = aMeta->maybeGetAttr("description"))
+ description = aDescription->getString();
+ }
+ */
+
+ logger->stdout("%s: %s '%s'",
+ headerPrefix,
+ attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" :
+ attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" :
+ attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" :
+ "package",
+ name);
+ };
+
+ if (attrPath.size() == 0
+ || (attrPath.size() == 1 && (
+ attrPath[0] == "defaultPackage"
+ || attrPath[0] == "devShell"
+ || attrPath[0] == "nixosConfigurations"
+ || attrPath[0] == "nixosModules"
+ || attrPath[0] == "defaultApp"
+ || attrPath[0] == "templates"))
+ || ((attrPath.size() == 1 || attrPath.size() == 2)
+ && (attrPath[0] == "checks"
+ || attrPath[0] == "packages"
+ || attrPath[0] == "apps"))
+ )
+ {
+ recurse();
+ }
+
+ else if (
+ (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell"))
+ || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages"))
+ )
+ {
+ if (visitor.isDerivation())
+ showDerivation();
+ else
+ throw Error("expected a derivation");
+ }
+
+ else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") {
+ if (visitor.isDerivation())
+ showDerivation();
+ else
+ recurse();
+ }
+
+ else if (attrPath.size() > 0 && attrPath[0] == "legacyPackages") {
+ if (attrPath.size() == 1)
+ recurse();
+ else if (!showLegacy)
+ logger->stdout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
+ else {
+ if (visitor.isDerivation())
+ showDerivation();
+ else if (attrPath.size() <= 2)
+ // FIXME: handle recurseIntoAttrs
+ recurse();
+ }
+ }
+
+ else if (
+ (attrPath.size() == 2 && attrPath[0] == "defaultApp") ||
+ (attrPath.size() == 3 && attrPath[0] == "apps"))
+ {
+ auto aType = visitor.maybeGetAttr("type");
+ if (!aType || aType->getString() != "app")
+ throw EvalError("not an app definition");
+ logger->stdout("%s: app", headerPrefix);
+ }
+
+ else if (
+ (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") ||
+ (attrPath.size() == 2 && attrPath[0] == "templates"))
+ {
+ auto description = visitor.getAttr("description")->getString();
+ logger->stdout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
+ }
+
+ else {
+ logger->stdout("%s: %s",
+ headerPrefix,
+ attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" :
+ attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" :
+ attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" :
+ ANSI_YELLOW "unknown" ANSI_NORMAL);
+ }
+ } catch (EvalError & e) {
+ if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
+ throw;
+ }
+ };
+
+ auto cache = openEvalCache(*state, flake, useEvalCache);
+
+ visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
+ }
+};
+
+struct CmdFlake : virtual MultiCommand, virtual Command
+{
+ CmdFlake()
+ : MultiCommand({
+ {"update", []() { return make_ref<CmdFlakeUpdate>(); }},
+ {"info", []() { return make_ref<CmdFlakeInfo>(); }},
+ {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }},
+ {"check", []() { return make_ref<CmdFlakeCheck>(); }},
+ {"init", []() { return make_ref<CmdFlakeInit>(); }},
+ {"new", []() { return make_ref<CmdFlakeNew>(); }},
+ {"clone", []() { return make_ref<CmdFlakeClone>(); }},
+ {"archive", []() { return make_ref<CmdFlakeArchive>(); }},
+ {"show", []() { return make_ref<CmdFlakeShow>(); }},
+ })
+ {
+ }
+
+ std::string description() override
+ {
+ return "manage Nix flakes";
+ }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix flake' requires a sub-command.");
+ settings.requireExperimentalFeature("flakes");
+ command->second->prepare();
+ command->second->run();
+ }
+
+ void printHelp(const string & programName, std::ostream & out) override
+ {
+ MultiCommand::printHelp(programName, out);
+ }
+};
+
+static auto r1 = registerCommand<CmdFlake>("flake");
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index a25ec43a9..2e0e83561 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -1,9 +1,18 @@
set -e
if [ -e .attrs.sh ]; then source .attrs.sh; fi
+
+outputs=$_outputs_saved
+for __output in $_outputs_saved; do
+ declare "$__output"="$out"
+done
+unset _outputs_saved __output
+
export IN_NIX_SHELL=impure
export dontAddDisableDepTrack=1
+
if [[ -n $stdenv ]]; then
source $stdenv/setup
fi
+
export > $out
set >> $out
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index b97c6d21f..b94751e45 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -31,7 +31,11 @@ struct CmdHash : Command
.labels({"modulus"})
.dest(&modulus);
#endif
- expectArgs("paths", &paths);
+ expectArgs({
+ .label = "paths",
+ .handler = {&paths},
+ .completer = completePath
+ });
}
std::string description() override
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 708a0dc88..babb0e5fe 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -1,3 +1,4 @@
+#include "installables.hh"
#include "command.hh"
#include "attr-path.hh"
#include "common-eval-args.hh"
@@ -7,80 +8,262 @@
#include "get-drvs.hh"
#include "store-api.hh"
#include "shared.hh"
+#include "flake/flake.hh"
+#include "eval-cache.hh"
+#include "url.hh"
+#include "registry.hh"
#include <regex>
+#include <queue>
namespace nix {
+void completeFlakeInputPath(
+ ref<EvalState> evalState,
+ const FlakeRef & flakeRef,
+ std::string_view prefix)
+{
+ auto flake = flake::getFlake(*evalState, flakeRef, true);
+ for (auto & input : flake.inputs)
+ if (hasPrefix(input.first, prefix))
+ completions->insert(input.first);
+}
-SourceExprCommand::SourceExprCommand()
+MixFlakeOptions::MixFlakeOptions()
{
addFlag({
- .longName = "file",
- .shortName = 'f',
- .description = "evaluate FILE rather than the default",
- .labels = {"file"},
- .handler = {&file}
+ .longName = "recreate-lock-file",
+ .description = "recreate lock file from scratch",
+ .handler = {&lockFlags.recreateLockFile, true}
});
-}
-Value * SourceExprCommand::getSourceExpr(EvalState & state)
-{
- if (vSourceExpr) return *vSourceExpr;
+ addFlag({
+ .longName = "no-update-lock-file",
+ .description = "do not allow any updates to the lock file",
+ .handler = {&lockFlags.updateLockFile, false}
+ });
- auto sToplevel = state.symbols.create("_toplevel");
+ addFlag({
+ .longName = "no-write-lock-file",
+ .description = "do not write the newly generated lock file",
+ .handler = {&lockFlags.writeLockFile, false}
+ });
- vSourceExpr = allocRootValue(state.allocValue());
+ addFlag({
+ .longName = "no-registries",
+ .description = "don't use flake registries",
+ .handler = {&lockFlags.useRegistries, false}
+ });
- if (file != "")
- state.evalFile(lookupFileArg(state, file), **vSourceExpr);
+ addFlag({
+ .longName = "commit-lock-file",
+ .description = "commit changes to the lock file",
+ .handler = {&lockFlags.commitLockFile, true}
+ });
- else {
+ addFlag({
+ .longName = "update-input",
+ .description = "update a specific flake input",
+ .labels = {"input-path"},
+ .handler = {[&](std::string s) {
+ lockFlags.inputUpdates.insert(flake::parseInputPath(s));
+ }},
+ .completer = {[&](size_t, std::string_view prefix) {
+ if (auto flakeRef = getFlakeRefForCompletion())
+ completeFlakeInputPath(getEvalState(), *flakeRef, prefix);
+ }}
+ });
- /* Construct the installation source from $NIX_PATH. */
+ addFlag({
+ .longName = "override-input",
+ .description = "override a specific flake input (e.g. 'dwarffs/nixpkgs')",
+ .labels = {"input-path", "flake-url"},
+ .handler = {[&](std::string inputPath, std::string flakeRef) {
+ lockFlags.inputOverrides.insert_or_assign(
+ flake::parseInputPath(inputPath),
+ parseFlakeRef(flakeRef, absPath(".")));
+ }}
+ });
- auto searchPath = state.getSearchPath();
+ addFlag({
+ .longName = "inputs-from",
+ .description = "use the inputs of the specified flake as registry entries",
+ .labels = {"flake-url"},
+ .handler = {[&](std::string flakeRef) {
+ auto evalState = getEvalState();
+ auto flake = flake::lockFlake(
+ *evalState,
+ parseFlakeRef(flakeRef, absPath(".")),
+ { .writeLockFile = false });
+ for (auto & [inputName, input] : flake.lockFile.root->inputs) {
+ auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes
+ if (auto input3 = std::dynamic_pointer_cast<const flake::LockedNode>(input2)) {
+ overrideRegistry(
+ fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}),
+ input3->lockedRef.input,
+ {});
+ }
+ }
+ }},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeFlakeRef(getEvalState()->store, prefix);
+ }}
+ });
+}
- state.mkAttrs(**vSourceExpr, 1024);
+SourceExprCommand::SourceExprCommand()
+{
+ addFlag({
+ .longName = "file",
+ .shortName = 'f',
+ .description = "evaluate FILE rather than the default",
+ .labels = {"file"},
+ .handler = {&file},
+ .completer = completePath
+ });
- mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true);
+ addFlag({
+ .longName ="expr",
+ .description = "evaluate attributes from EXPR",
+ .labels = {"expr"},
+ .handler = {&expr}
+ });
- std::unordered_set<std::string> seen;
+ addFlag({
+ .longName ="derivation",
+ .description = "operate on the store derivation rather than its outputs",
+ .handler = {&operateOn, OperateOn::Derivation},
+ });
+}
- auto addEntry = [&](const std::string & name) {
- if (name == "") return;
- if (!seen.insert(name).second) return;
- Value * v1 = state.allocValue();
- mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath"));
- Value * v2 = state.allocValue();
- mkApp(*v2, *v1, mkString(*state.allocValue(), name));
- mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)),
- state.getBuiltin("import"), *v2);
- };
+Strings SourceExprCommand::getDefaultFlakeAttrPaths()
+{
+ return {"defaultPackage." + settings.thisSystem.get()};
+}
- for (auto & i : searchPath)
- /* Hack to handle channels. */
- if (i.first.empty() && pathExists(i.second + "/manifest.nix")) {
- for (auto & j : readDirectory(i.second))
- if (j.name != "manifest.nix"
- && pathExists(fmt("%s/%s/default.nix", i.second, j.name)))
- addEntry(j.name);
- } else
- addEntry(i.first);
+Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
+{
+ return {
+ // As a convenience, look for the attribute in
+ // 'outputs.packages'.
+ "packages." + settings.thisSystem.get() + ".",
+ // As a temporary hack until Nixpkgs is properly converted
+ // to provide a clean 'packages' set, look in 'legacyPackages'.
+ "legacyPackages." + settings.thisSystem.get() + "."
+ };
+}
+
+void SourceExprCommand::completeInstallable(std::string_view prefix)
+{
+ if (file) return; // FIXME
+
+ completeFlakeRefWithFragment(
+ getEvalState(),
+ lockFlags,
+ getDefaultFlakeAttrPathPrefixes(),
+ getDefaultFlakeAttrPaths(),
+ prefix);
+}
- (*vSourceExpr)->attrs->sort();
+void completeFlakeRefWithFragment(
+ ref<EvalState> evalState,
+ flake::LockFlags lockFlags,
+ Strings attrPathPrefixes,
+ const Strings & defaultFlakeAttrPaths,
+ std::string_view prefix)
+{
+ /* Look for flake output attributes that match the
+ prefix. */
+ try {
+ auto hash = prefix.find('#');
+ if (hash != std::string::npos) {
+ auto fragment = prefix.substr(hash + 1);
+ auto flakeRefS = std::string(prefix.substr(0, hash));
+ // FIXME: do tilde expansion.
+ auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
+
+ auto evalCache = openEvalCache(*evalState,
+ std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)),
+ true);
+
+ auto root = evalCache->getRoot();
+
+ /* Complete 'fragment' relative to all the
+ attrpath prefixes as well as the root of the
+ flake. */
+ attrPathPrefixes.push_back("");
+
+ for (auto & attrPathPrefixS : attrPathPrefixes) {
+ auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS);
+ auto attrPathS = attrPathPrefixS + std::string(fragment);
+ auto attrPath = parseAttrPath(*evalState, attrPathS);
+
+ std::string lastAttr;
+ if (!attrPath.empty() && !hasSuffix(attrPathS, ".")) {
+ lastAttr = attrPath.back();
+ attrPath.pop_back();
+ }
+
+ auto attr = root->findAlongAttrPath(attrPath);
+ if (!attr) continue;
+
+ for (auto & attr2 : attr->getAttrs()) {
+ if (hasPrefix(attr2, lastAttr)) {
+ auto attrPath2 = attr->getAttrPath(attr2);
+ /* Strip the attrpath prefix. */
+ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
+ completions->insert(flakeRefS + "#" + concatStringsSep(".", attrPath2));
+ }
+ }
+ }
+
+ /* And add an empty completion for the default
+ attrpaths. */
+ if (fragment.empty()) {
+ for (auto & attrPath : defaultFlakeAttrPaths) {
+ auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
+ if (!attr) continue;
+ completions->insert(flakeRefS + "#");
+ }
+ }
+ }
+ } catch (Error & e) {
+ warn(e.msg());
}
- return *vSourceExpr;
+ completeFlakeRef(evalState->store, prefix);
}
-ref<EvalState> SourceExprCommand::getEvalState()
+ref<EvalState> EvalCommand::getEvalState()
{
if (!evalState)
evalState = std::make_shared<EvalState>(searchPath, getStore());
return ref<EvalState>(evalState);
}
+void completeFlakeRef(ref<Store> store, std::string_view prefix)
+{
+ if (prefix == "")
+ completions->insert(".");
+
+ completeDir(0, prefix);
+
+ /* Look for registry entries that match the prefix. */
+ for (auto & registry : fetchers::getRegistries(store)) {
+ for (auto & entry : registry->entries) {
+ auto from = entry.from.to_string();
+ if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
+ std::string from2(from, 6);
+ if (hasPrefix(from2, prefix))
+ completions->insert(from2);
+ } else {
+ if (hasPrefix(from, prefix))
+ completions->insert(from);
+ }
+ }
+ }
+}
+
Buildable Installable::toBuildable()
{
auto buildables = toBuildables();
@@ -89,27 +272,55 @@ Buildable Installable::toBuildable()
return std::move(buildables[0]);
}
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+Installable::getCursors(EvalState & state, bool useEvalCache)
+{
+ auto evalCache =
+ std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
+ [&]() { return toValue(state).first; });
+ return {{evalCache->getRoot(), ""}};
+}
+
+std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+Installable::getCursor(EvalState & state, bool useEvalCache)
+{
+ auto cursors = getCursors(state, useEvalCache);
+ if (cursors.empty())
+ throw Error("cannot find flake attribute '%s'", what());
+ return cursors[0];
+}
+
struct InstallableStorePath : Installable
{
ref<Store> store;
StorePath storePath;
- InstallableStorePath(ref<Store> store, const Path & storePath)
- : store(store), storePath(store->parseStorePath(storePath)) { }
+ InstallableStorePath(ref<Store> store, StorePath && storePath)
+ : store(store), storePath(std::move(storePath)) { }
std::string what() override { return store->printStorePath(storePath); }
Buildables toBuildables() override
{
- std::map<std::string, StorePath> outputs;
- outputs.insert_or_assign("out", storePath);
- Buildable b{
- .drvPath = storePath.isDerivation() ? storePath : std::optional<StorePath>(),
- .outputs = std::move(outputs)
- };
- Buildables bs;
- bs.push_back(std::move(b));
- return bs;
+ if (storePath.isDerivation()) {
+ std::map<std::string, StorePath> outputs;
+ auto drv = store->readDerivation(storePath);
+ for (auto & [name, output] : drv.outputs)
+ outputs.emplace(name, output.path(*store, drv.name));
+ return {
+ Buildable {
+ .drvPath = storePath,
+ .outputs = std::move(outputs)
+ }
+ };
+ } else {
+ return {
+ Buildable {
+ .drvPath = {},
+ .outputs = {{"out", storePath}}
+ }
+ };
+ }
}
std::optional<StorePath> getStorePath() override
@@ -118,146 +329,325 @@ struct InstallableStorePath : Installable
}
};
-struct InstallableValue : Installable
+Buildables InstallableValue::toBuildables()
+{
+ Buildables res;
+
+ StorePathSet drvPaths;
+
+ for (auto & drv : toDerivations()) {
+ Buildable b{.drvPath = drv.drvPath};
+ drvPaths.insert(drv.drvPath);
+
+ auto outputName = drv.outputName;
+ if (outputName == "")
+ throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath));
+
+ b.outputs.emplace(outputName, drv.outPath);
+
+ res.push_back(std::move(b));
+ }
+
+ // Hack to recognize .all: if all drvs have the same drvPath,
+ // merge the buildables.
+ if (drvPaths.size() == 1) {
+ Buildable b{.drvPath = *drvPaths.begin()};
+ for (auto & b2 : res)
+ for (auto & output : b2.outputs)
+ b.outputs.insert_or_assign(output.first, output.second);
+ Buildables bs;
+ bs.push_back(std::move(b));
+ return bs;
+ } else
+ return res;
+}
+
+struct InstallableAttrPath : InstallableValue
{
SourceExprCommand & cmd;
+ RootValue v;
+ std::string attrPath;
- InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { }
+ InstallableAttrPath(ref<EvalState> state, SourceExprCommand & cmd, Value * v, const std::string & attrPath)
+ : InstallableValue(state), cmd(cmd), v(allocRootValue(v)), attrPath(attrPath)
+ { }
- Buildables toBuildables() override
+ std::string what() override { return attrPath; }
+
+ std::pair<Value *, Pos> toValue(EvalState & state) override
{
- auto state = cmd.getEvalState();
+ auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), **v);
+ state.forceValue(*vRes);
+ return {vRes, pos};
+ }
- auto v = toValue(*state).first;
+ virtual std::vector<InstallableValue::DerivationInfo> toDerivations() override;
+};
- Bindings & autoArgs = *cmd.getAutoArgs(*state);
+std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations()
+{
+ auto v = toValue(*state).first;
- DrvInfos drvs;
- getDerivations(*state, *v, "", autoArgs, drvs, false);
+ Bindings & autoArgs = *cmd.getAutoArgs(*state);
- Buildables res;
+ DrvInfos drvInfos;
+ getDerivations(*state, *v, "", autoArgs, drvInfos, false);
- StorePathSet drvPaths;
+ std::vector<DerivationInfo> res;
+ for (auto & drvInfo : drvInfos) {
+ res.push_back({
+ state->store->parseStorePath(drvInfo.queryDrvPath()),
+ state->store->parseStorePath(drvInfo.queryOutPath()),
+ drvInfo.queryOutputName()
+ });
+ }
- for (auto & drv : drvs) {
- Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())};
- drvPaths.insert(*b.drvPath);
+ return res;
+}
- auto outputName = drv.queryOutputName();
- if (outputName == "")
- throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath));
+std::vector<std::string> InstallableFlake::getActualAttrPaths()
+{
+ std::vector<std::string> res;
- b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath()));
+ for (auto & prefix : prefixes)
+ res.push_back(prefix + *attrPaths.begin());
- res.push_back(std::move(b));
- }
+ for (auto & s : attrPaths)
+ res.push_back(s);
+
+ return res;
+}
+
+Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake)
+{
+ auto vFlake = state.allocValue();
+
+ callFlake(state, lockedFlake, *vFlake);
- // Hack to recognize .all: if all drvs have the same drvPath,
- // merge the buildables.
- if (drvPaths.size() == 1) {
- Buildable b{.drvPath = *drvPaths.begin()};
- for (auto & b2 : res)
- for (auto & output : b2.outputs)
- b.outputs.insert_or_assign(output.first, output.second);
- Buildables bs;
- bs.push_back(std::move(b));
- return bs;
- } else
- return res;
+ auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+ assert(aOutputs);
+
+ state.forceValue(*aOutputs->value);
+
+ return aOutputs->value;
+}
+
+ref<eval_cache::EvalCache> openEvalCache(
+ EvalState & state,
+ std::shared_ptr<flake::LockedFlake> lockedFlake,
+ bool useEvalCache)
+{
+ auto fingerprint = lockedFlake->getFingerprint();
+ return make_ref<nix::eval_cache::EvalCache>(
+ useEvalCache && evalSettings.pureEval
+ ? std::optional { std::cref(fingerprint) }
+ : std::nullopt,
+ state,
+ [&state, lockedFlake]()
+ {
+ /* For testing whether the evaluation cache is
+ complete. */
+ if (getEnv("NIX_ALLOW_EVAL").value_or("1") == "0")
+ throw Error("not everything is cached, but evaluation is not allowed");
+
+ auto vFlake = state.allocValue();
+ flake::callFlake(state, *lockedFlake, *vFlake);
+
+ state.forceAttrs(*vFlake);
+
+ auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs"));
+ assert(aOutputs);
+
+ return aOutputs->value;
+ });
+}
+
+static std::string showAttrPaths(const std::vector<std::string> & paths)
+{
+ std::string s;
+ for (const auto & [n, i] : enumerate(paths)) {
+ if (n > 0) s += n + 1 == paths.size() ? " or " : ", ";
+ s += '\''; s += i; s += '\'';
}
-};
+ return s;
+}
-struct InstallableExpr : InstallableValue
+std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
{
- std::string text;
- InstallableExpr(SourceExprCommand & cmd, const std::string & text)
- : InstallableValue(cmd), text(text) { }
+ auto lockedFlake = getLockedFlake();
- std::string what() override { return text; }
+ auto cache = openEvalCache(*state, lockedFlake, true);
+ auto root = cache->getRoot();
- std::pair<Value *, Pos> toValue(EvalState & state) override
- {
- auto v = state.allocValue();
- state.eval(state.parseExprFromString(text, absPath(".")), *v);
- return {v, noPos};
+ for (auto & attrPath : getActualAttrPaths()) {
+ auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
+ if (!attr) continue;
+
+ if (!attr->isDerivation())
+ throw Error("flake output attribute '%s' is not a derivation", attrPath);
+
+ auto drvPath = attr->forceDerivation();
+
+ auto drvInfo = DerivationInfo{
+ std::move(drvPath),
+ state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
+ attr->getAttr(state->sOutputName)->getString()
+ };
+
+ return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)};
}
-};
-struct InstallableAttrPath : InstallableValue
+ throw Error("flake '%s' does not provide attribute %s",
+ flakeRef, showAttrPaths(getActualAttrPaths()));
+}
+
+std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations()
{
- std::string attrPath;
+ std::vector<DerivationInfo> res;
+ res.push_back(std::get<2>(toDerivation()));
+ return res;
+}
- InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath)
- : InstallableValue(cmd), attrPath(attrPath)
- { }
+std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
+{
+ auto lockedFlake = getLockedFlake();
- std::string what() override { return attrPath; }
+ auto vOutputs = getFlakeOutputs(state, *lockedFlake);
- std::pair<Value *, Pos> toValue(EvalState & state) override
- {
- auto source = cmd.getSourceExpr(state);
+ auto emptyArgs = state.allocBindings(0);
- Bindings & autoArgs = *cmd.getAutoArgs(state);
+ for (auto & attrPath : getActualAttrPaths()) {
+ try {
+ auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs);
+ state.forceValue(*v);
+ return {v, pos};
+ } catch (AttrPathNotFound & e) {
+ }
+ }
- auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first;
- state.forceValue(*v);
+ throw Error("flake '%s' does not provide attribute %s",
+ flakeRef, showAttrPaths(getActualAttrPaths()));
+}
- return {v, noPos};
+std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+InstallableFlake::getCursors(EvalState & state, bool useEvalCache)
+{
+ auto evalCache = openEvalCache(state,
+ std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)),
+ useEvalCache);
+
+ auto root = evalCache->getRoot();
+
+ std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res;
+
+ for (auto & attrPath : getActualAttrPaths()) {
+ auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath));
+ if (attr) res.push_back({attr, attrPath});
}
-};
-// FIXME: extend
-std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)";
-static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex));
+ return res;
+}
+
+std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
+{
+ if (!_lockedFlake)
+ _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
+ return _lockedFlake;
+}
-static std::vector<std::shared_ptr<Installable>> parseInstallables(
- SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables)
+FlakeRef InstallableFlake::nixpkgsFlakeRef() const
{
- std::vector<std::shared_ptr<Installable>> result;
+ auto lockedFlake = getLockedFlake();
- if (ss.empty() && useDefaultInstallables) {
- if (cmd.file == "")
- cmd.file = ".";
- ss = {""};
+ if (auto nixpkgsInput = lockedFlake->lockFile.findInput({"nixpkgs"})) {
+ if (auto lockedNode = std::dynamic_pointer_cast<const flake::LockedNode>(nixpkgsInput)) {
+ debug("using nixpkgs flake '%s'", lockedNode->lockedRef);
+ return std::move(lockedNode->lockedRef);
+ }
}
- for (auto & s : ss) {
+ return Installable::nixpkgsFlakeRef();
+}
+
+std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
+ ref<Store> store, std::vector<std::string> ss)
+{
+ std::vector<std::shared_ptr<Installable>> result;
- if (s.compare(0, 1, "(") == 0)
- result.push_back(std::make_shared<InstallableExpr>(cmd, s));
+ if (file || expr) {
+ if (file && expr)
+ throw UsageError("'--file' and '--expr' are exclusive");
- else if (s.find("/") != std::string::npos) {
+ // FIXME: backward compatibility hack
+ if (file) evalSettings.pureEval = false;
- auto path = store->toStorePath(store->followLinksToStore(s));
+ auto state = getEvalState();
+ auto vFile = state->allocValue();
- if (store->isStorePath(path))
- result.push_back(std::make_shared<InstallableStorePath>(store, path));
+ if (file)
+ state->evalFile(lookupFileArg(*state, *file), *vFile);
+ else {
+ auto e = state->parseExprFromString(*expr, absPath("."));
+ state->eval(e, *vFile);
}
- else if (s == "" || std::regex_match(s, attrPathRegex))
- result.push_back(std::make_shared<InstallableAttrPath>(cmd, s));
+ for (auto & s : ss)
+ result.push_back(std::make_shared<InstallableAttrPath>(state, *this, vFile, s == "." ? "" : s));
+
+ } else {
+
+ for (auto & s : ss) {
+ std::exception_ptr ex;
+
+ try {
+ auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
+ result.push_back(std::make_shared<InstallableFlake>(
+ getEvalState(), std::move(flakeRef),
+ fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
+ getDefaultFlakeAttrPathPrefixes(), lockFlags));
+ continue;
+ } catch (...) {
+ ex = std::current_exception();
+ }
+
+ if (s.find('/') != std::string::npos) {
+ try {
+ result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
+ continue;
+ } catch (BadStorePath &) {
+ } catch (...) {
+ if (!ex)
+ ex = std::current_exception();
+ }
+ }
+
+ std::rethrow_exception(ex);
- else
- throw UsageError("don't know what to do with argument '%s'", s);
+ /*
+ throw Error(
+ pathExists(s)
+ ? "path '%s' is not a flake or a store path"
+ : "don't know how to handle argument '%s'", s);
+ */
+ }
}
return result;
}
-std::shared_ptr<Installable> parseInstallable(
- SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
- bool useDefaultInstallables)
+std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
+ ref<Store> store, const std::string & installable)
{
- auto installables = parseInstallables(cmd, store, {installable}, false);
+ auto installables = parseInstallables(store, {installable});
assert(installables.size() == 1);
return installables.front();
}
-Buildables build(ref<Store> store, RealiseMode mode,
- std::vector<std::shared_ptr<Installable>> installables)
+Buildables build(ref<Store> store, Realise mode,
+ std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
{
- if (mode != Build)
+ if (mode == Realise::Nothing)
settings.readOnlyMode = true;
Buildables buildables;
@@ -278,33 +668,45 @@ Buildables build(ref<Store> store, RealiseMode mode,
}
}
- if (mode == DryRun)
+ if (mode == Realise::Nothing)
printMissing(store, pathsToBuild, lvlError);
- else if (mode == Build)
- store->buildPaths(pathsToBuild);
+ else if (mode == Realise::Outputs)
+ store->buildPaths(pathsToBuild, bMode);
return buildables;
}
-StorePathSet toStorePaths(ref<Store> store, RealiseMode mode,
+StorePathSet toStorePaths(ref<Store> store,
+ Realise mode, OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
{
StorePathSet outPaths;
- for (auto & b : build(store, mode, installables))
- for (auto & output : b.outputs)
- outPaths.insert(output.second);
+ if (operateOn == OperateOn::Output) {
+ for (auto & b : build(store, mode, installables))
+ for (auto & output : b.outputs)
+ outPaths.insert(output.second);
+ } else {
+ if (mode == Realise::Nothing)
+ settings.readOnlyMode = true;
+
+ for (auto & i : installables)
+ for (auto & b : i->toBuildables())
+ if (b.drvPath)
+ outPaths.insert(*b.drvPath);
+ }
return outPaths;
}
-StorePath toStorePath(ref<Store> store, RealiseMode mode,
+StorePath toStorePath(ref<Store> store,
+ Realise mode, OperateOn operateOn,
std::shared_ptr<Installable> installable)
{
- auto paths = toStorePaths(store, mode, {installable});
+ auto paths = toStorePaths(store, mode, operateOn, {installable});
if (paths.size() != 1)
- throw Error("argument '%s' should evaluate to one store path", installable->what());
+ throw Error("argument '%s' should evaluate to one store path", installable->what());
return *paths.begin();
}
@@ -333,14 +735,51 @@ StorePathSet toDerivations(ref<Store> store,
return drvPaths;
}
+InstallablesCommand::InstallablesCommand()
+{
+ expectArgs({
+ .label = "installables",
+ .handler = {&_installables},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeInstallable(prefix);
+ }}
+ });
+}
+
void InstallablesCommand::prepare()
{
- installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables());
+ if (_installables.empty() && useDefaultInstallables())
+ // FIXME: commands like "nix install" should not have a
+ // default, probably.
+ _installables.push_back(".");
+ installables = parseInstallables(getStore(), _installables);
+}
+
+std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion()
+{
+ if (_installables.empty()) {
+ if (useDefaultInstallables())
+ return parseFlakeRef(".", absPath("."));
+ return {};
+ }
+ return parseFlakeRef(_installables.front(), absPath("."));
+}
+
+InstallableCommand::InstallableCommand()
+{
+ expectArgs({
+ .label = "installable",
+ .optional = true,
+ .handler = {&_installable},
+ .completer = {[&](size_t, std::string_view prefix) {
+ completeInstallable(prefix);
+ }}
+ });
}
void InstallableCommand::prepare()
{
- installable = parseInstallable(*this, getStore(), _installable, false);
+ installable = parseInstallable(getStore(), _installable);
}
}
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index 503984220..eb34365d4 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -3,11 +3,17 @@
#include "util.hh"
#include "path.hh"
#include "eval.hh"
+#include "flake/flake.hh"
#include <optional>
namespace nix {
+struct DrvInfo;
+struct SourceExprCommand;
+
+namespace eval_cache { class EvalCache; class AttrCursor; }
+
struct Buildable
{
std::optional<StorePath> drvPath;
@@ -16,19 +22,25 @@ struct Buildable
typedef std::vector<Buildable> Buildables;
+struct App
+{
+ std::vector<StorePathWithOutputs> context;
+ Path program;
+ // FIXME: add args, sandbox settings, metadata, ...
+};
+
struct Installable
{
virtual ~Installable() { }
virtual std::string what() = 0;
- virtual Buildables toBuildables()
- {
- throw Error("argument '%s' cannot be built", what());
- }
+ virtual Buildables toBuildables() = 0;
Buildable toBuildable();
+ App toApp(EvalState & state);
+
virtual std::pair<Value *, Pos> toValue(EvalState & state)
{
throw Error("argument '%s' cannot be evaluated", what());
@@ -40,6 +52,74 @@ struct Installable
{
return {};
}
+
+ virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+ getCursors(EvalState & state, bool useEvalCache);
+
+ std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
+ getCursor(EvalState & state, bool useEvalCache);
+
+ virtual FlakeRef nixpkgsFlakeRef() const
+ {
+ return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}));
+ }
};
+struct InstallableValue : Installable
+{
+ ref<EvalState> state;
+
+ InstallableValue(ref<EvalState> state) : state(state) {}
+
+ struct DerivationInfo
+ {
+ StorePath drvPath;
+ StorePath outPath;
+ std::string outputName;
+ };
+
+ virtual std::vector<DerivationInfo> toDerivations() = 0;
+
+ Buildables toBuildables() override;
+};
+
+struct InstallableFlake : InstallableValue
+{
+ FlakeRef flakeRef;
+ Strings attrPaths;
+ Strings prefixes;
+ const flake::LockFlags & lockFlags;
+ mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
+
+ InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef,
+ Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags)
+ : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths),
+ prefixes(prefixes), lockFlags(lockFlags)
+ { }
+
+ std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
+
+ std::vector<std::string> getActualAttrPaths();
+
+ Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake);
+
+ std::tuple<std::string, FlakeRef, DerivationInfo> toDerivation();
+
+ std::vector<DerivationInfo> toDerivations() override;
+
+ std::pair<Value *, Pos> toValue(EvalState & state) override;
+
+ std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
+ getCursors(EvalState & state, bool useEvalCache) override;
+
+ std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
+
+ FlakeRef nixpkgsFlakeRef() const override;
+};
+
+ref<eval_cache::EvalCache> openEvalCache(
+ EvalState & state,
+ std::shared_ptr<flake::LockedFlake> lockedFlake,
+ bool useEvalCache);
+
}
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 3fe22f6c2..7e10d373a 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -18,7 +18,7 @@ struct CmdLog : InstallableCommand
return {
Example{
"To get the build log of GNU Hello:",
- "nix log nixpkgs.hello"
+ "nix log nixpkgs#hello"
},
Example{
"To get the build log of a specific path:",
@@ -26,7 +26,7 @@ struct CmdLog : InstallableCommand
},
Example{
"To get a build log from a specific binary cache:",
- "nix log --store https://cache.nixos.org nixpkgs.hello"
+ "nix log --store https://cache.nixos.org nixpkgs#hello"
},
};
}
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index d2157f2d4..76c8bc9a3 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -85,7 +85,11 @@ struct CmdLsStore : StoreCommand, MixLs
{
CmdLsStore()
{
- expectArg("path", &path);
+ expectArgs({
+ .label = "path",
+ .handler = {&path},
+ .completer = completePath
+ });
}
Examples examples() override
@@ -117,7 +121,11 @@ struct CmdLsNar : Command, MixLs
CmdLsNar()
{
- expectArg("nar", &narPath);
+ expectArgs({
+ .label = "nar",
+ .handler = {&narPath},
+ .completer = completePath
+ });
expectArg("path", &path);
}
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 203901168..e62657e95 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -7,7 +7,6 @@
#include "legacy.hh"
#include "shared.hh"
#include "store-api.hh"
-#include "progress-bar.hh"
#include "filetransfer.hh"
#include "finally.hh"
#include "loggers.hh"
@@ -69,7 +68,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
addFlag({
.longName = "help",
.description = "show usage information",
- .handler = {[&]() { showHelpAndExit(); }},
+ .handler = {[&]() { if (!completions) showHelpAndExit(); }},
});
addFlag({
@@ -97,7 +96,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
addFlag({
.longName = "version",
.description = "show version information",
- .handler = {[&]() { printVersion(programName); }},
+ .handler = {[&]() { if (!completions) printVersion(programName); }},
});
addFlag({
@@ -165,6 +164,7 @@ void mainWrapped(int argc, char * * argv)
verbosity = lvlWarn;
settings.verboseBuild = false;
+ evalSettings.pureEval = true;
setLogFormat("bar");
@@ -172,7 +172,22 @@ void mainWrapped(int argc, char * * argv)
NixArgs args;
- args.parseCmdline(argvToStrings(argc, argv));
+ Finally printCompletions([&]()
+ {
+ if (completions) {
+ std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n");
+ for (auto & s : *completions)
+ std::cout << s << "\n";
+ }
+ });
+
+ try {
+ args.parseCmdline(argvToStrings(argc, argv));
+ } catch (UsageError &) {
+ if (!completions) throw;
+ }
+
+ if (completions) return;
initPlugins();
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index fb36fc410..2fe2e2fb2 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -10,7 +10,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
{
CmdMakeContentAddressable()
{
- realiseMode = Build;
+ realiseMode = Realise::Outputs;
}
std::string description() override
@@ -23,7 +23,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
return {
Example{
"To create a content-addressable representation of GNU Hello (but not its dependencies):",
- "nix make-content-addressable nixpkgs.hello"
+ "nix make-content-addressable nixpkgs#hello"
},
Example{
"To compute a content-addressable representation of the current NixOS system closure:",
@@ -84,7 +84,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
info.narSize = sink.s->size();
info.ca = FixedOutputHash {
.method = FileIngestionMethod::Recursive,
- .hash = info.narHash,
+ .hash = *info.narHash,
};
if (!json)
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index b89a44f83..0c12efaf0 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -40,7 +40,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
},
Example{
"To show a package's closure size and all its dependencies with human readable sizes:",
- "nix path-info -rsSh nixpkgs.rust"
+ "nix path-info -rsSh nixpkgs#rust"
},
Example{
"To check the existence of a path in a binary cache:",
@@ -61,7 +61,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
};
}
- void printSize(unsigned long long value)
+ void printSize(uint64_t value)
{
if (!humanReadable) {
std::cout << fmt("\t%11d", value);
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
new file mode 100644
index 000000000..7dcc0b6d4
--- /dev/null
+++ b/src/nix/profile.cc
@@ -0,0 +1,469 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "derivations.hh"
+#include "archive.hh"
+#include "builtins/buildenv.hh"
+#include "flake/flakeref.hh"
+#include "../nix-env/user-env.hh"
+#include "profiles.hh"
+
+#include <nlohmann/json.hpp>
+#include <regex>
+
+using namespace nix;
+
+struct ProfileElementSource
+{
+ FlakeRef originalRef;
+ // FIXME: record original attrpath.
+ FlakeRef resolvedRef;
+ std::string attrPath;
+ // FIXME: output names
+};
+
+struct ProfileElement
+{
+ StorePathSet storePaths;
+ std::optional<ProfileElementSource> source;
+ bool active = true;
+ // FIXME: priority
+};
+
+struct ProfileManifest
+{
+ std::vector<ProfileElement> elements;
+
+ ProfileManifest() { }
+
+ ProfileManifest(EvalState & state, const Path & profile)
+ {
+ auto manifestPath = profile + "/manifest.json";
+
+ if (pathExists(manifestPath)) {
+ auto json = nlohmann::json::parse(readFile(manifestPath));
+
+ auto version = json.value("version", 0);
+ if (version != 1)
+ throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version);
+
+ for (auto & e : json["elements"]) {
+ ProfileElement element;
+ for (auto & p : e["storePaths"])
+ element.storePaths.insert(state.store->parseStorePath((std::string) p));
+ element.active = e["active"];
+ if (e.value("uri", "") != "") {
+ element.source = ProfileElementSource{
+ parseFlakeRef(e["originalUri"]),
+ parseFlakeRef(e["uri"]),
+ e["attrPath"]
+ };
+ }
+ elements.emplace_back(std::move(element));
+ }
+ }
+
+ else if (pathExists(profile + "/manifest.nix")) {
+ // FIXME: needed because of pure mode; ugly.
+ if (state.allowedPaths) {
+ state.allowedPaths->insert(state.store->followLinksToStore(profile));
+ state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix"));
+ }
+
+ auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile));
+
+ for (auto & drvInfo : drvInfos) {
+ ProfileElement element;
+ element.storePaths = {state.store->parseStorePath(drvInfo.queryOutPath())};
+ elements.emplace_back(std::move(element));
+ }
+ }
+ }
+
+ std::string toJSON(Store & store) const
+ {
+ auto array = nlohmann::json::array();
+ for (auto & element : elements) {
+ auto paths = nlohmann::json::array();
+ for (auto & path : element.storePaths)
+ paths.push_back(store.printStorePath(path));
+ nlohmann::json obj;
+ obj["storePaths"] = paths;
+ obj["active"] = element.active;
+ if (element.source) {
+ obj["originalUri"] = element.source->originalRef.to_string();
+ obj["uri"] = element.source->resolvedRef.to_string();
+ obj["attrPath"] = element.source->attrPath;
+ }
+ array.push_back(obj);
+ }
+ nlohmann::json json;
+ json["version"] = 1;
+ json["elements"] = array;
+ return json.dump();
+ }
+
+ StorePath build(ref<Store> store)
+ {
+ auto tempDir = createTempDir();
+
+ StorePathSet references;
+
+ Packages pkgs;
+ for (auto & element : elements) {
+ for (auto & path : element.storePaths) {
+ if (element.active)
+ pkgs.emplace_back(store->printStorePath(path), true, 5);
+ references.insert(path);
+ }
+ }
+
+ buildProfile(tempDir, std::move(pkgs));
+
+ writeFile(tempDir + "/manifest.json", toJSON(*store));
+
+ /* Add the symlink tree to the store. */
+ StringSink sink;
+ dumpPath(tempDir, sink);
+
+ auto narHash = hashString(htSHA256, *sink.s);
+
+ ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references));
+ info.references = std::move(references);
+ info.narHash = narHash;
+ info.narSize = sink.s->size();
+ info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = *info.narHash };
+
+ auto source = StringSource { *sink.s };
+ store->addToStore(info, source);
+
+ return std::move(info.path);
+ }
+};
+
+struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
+{
+ std::string description() override
+ {
+ return "install a package into a profile";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To install a package from Nixpkgs:",
+ "nix profile install nixpkgs#hello"
+ },
+ Example{
+ "To install a package from a specific branch of Nixpkgs:",
+ "nix profile install nixpkgs/release-19.09#hello"
+ },
+ Example{
+ "To install a package from a specific revision of Nixpkgs:",
+ "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello"
+ },
+ };
+ }
+
+ void run(ref<Store> store) override
+ {
+ ProfileManifest manifest(*getEvalState(), *profile);
+
+ std::vector<StorePathWithOutputs> pathsToBuild;
+
+ for (auto & installable : installables) {
+ if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
+ auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
+
+ ProfileElement element;
+ element.storePaths = {drv.outPath}; // FIXME
+ element.source = ProfileElementSource{
+ installable2->flakeRef,
+ resolvedRef,
+ attrPath,
+ };
+
+ pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
+
+ manifest.elements.emplace_back(std::move(element));
+ } else
+ throw Error("'nix profile install' does not support argument '%s'", installable->what());
+ }
+
+ store->buildPaths(pathsToBuild);
+
+ updateProfile(manifest.build(store));
+ }
+};
+
+class MixProfileElementMatchers : virtual Args
+{
+ std::vector<std::string> _matchers;
+
+public:
+
+ MixProfileElementMatchers()
+ {
+ expectArgs("elements", &_matchers);
+ }
+
+ typedef std::variant<size_t, Path, std::regex> Matcher;
+
+ std::vector<Matcher> getMatchers(ref<Store> store)
+ {
+ std::vector<Matcher> res;
+
+ for (auto & s : _matchers) {
+ size_t n;
+ if (string2Int(s, n))
+ res.push_back(n);
+ else if (store->isStorePath(s))
+ res.push_back(s);
+ else
+ res.push_back(std::regex(s, std::regex::extended | std::regex::icase));
+ }
+
+ return res;
+ }
+
+ bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers)
+ {
+ for (auto & matcher : matchers) {
+ if (auto n = std::get_if<size_t>(&matcher)) {
+ if (*n == pos) return true;
+ } else if (auto path = std::get_if<Path>(&matcher)) {
+ if (element.storePaths.count(store.parseStorePath(*path))) return true;
+ } else if (auto regex = std::get_if<std::regex>(&matcher)) {
+ if (element.source
+ && std::regex_match(element.source->attrPath, *regex))
+ return true;
+ }
+ }
+
+ return false;
+ }
+};
+
+struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElementMatchers
+{
+ std::string description() override
+ {
+ return "remove packages from a profile";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To remove a package by attribute path:",
+ "nix profile remove packages.x86_64-linux.hello"
+ },
+ Example{
+ "To remove all packages:",
+ "nix profile remove '.*'"
+ },
+ Example{
+ "To remove a package by store path:",
+ "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10"
+ },
+ Example{
+ "To remove a package by position:",
+ "nix profile remove 3"
+ },
+ };
+ }
+
+ void run(ref<Store> store) override
+ {
+ ProfileManifest oldManifest(*getEvalState(), *profile);
+
+ auto matchers = getMatchers(store);
+
+ ProfileManifest newManifest;
+
+ for (size_t i = 0; i < oldManifest.elements.size(); ++i) {
+ auto & element(oldManifest.elements[i]);
+ if (!matches(*store, element, i, matchers))
+ newManifest.elements.push_back(std::move(element));
+ }
+
+ // FIXME: warn about unused matchers?
+
+ printInfo("removed %d packages, kept %d packages",
+ oldManifest.elements.size() - newManifest.elements.size(),
+ newManifest.elements.size());
+
+ updateProfile(newManifest.build(store));
+ }
+};
+
+struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProfileElementMatchers
+{
+ std::string description() override
+ {
+ return "upgrade packages using their most recent flake";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To upgrade all packages that were installed using a mutable flake reference:",
+ "nix profile upgrade '.*'"
+ },
+ Example{
+ "To upgrade a specific package:",
+ "nix profile upgrade packages.x86_64-linux.hello"
+ },
+ };
+ }
+
+ void run(ref<Store> store) override
+ {
+ ProfileManifest manifest(*getEvalState(), *profile);
+
+ auto matchers = getMatchers(store);
+
+ // FIXME: code duplication
+ std::vector<StorePathWithOutputs> pathsToBuild;
+
+ for (size_t i = 0; i < manifest.elements.size(); ++i) {
+ auto & element(manifest.elements[i]);
+ if (element.source
+ && !element.source->originalRef.input.isImmutable()
+ && matches(*store, element, i, matchers))
+ {
+ Activity act(*logger, lvlChatty, actUnknown,
+ fmt("checking '%s' for updates", element.source->attrPath));
+
+ InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags);
+
+ auto [attrPath, resolvedRef, drv] = installable.toDerivation();
+
+ if (element.source->resolvedRef == resolvedRef) continue;
+
+ printInfo("upgrading '%s' from flake '%s' to '%s'",
+ element.source->attrPath, element.source->resolvedRef, resolvedRef);
+
+ element.storePaths = {drv.outPath}; // FIXME
+ element.source = ProfileElementSource{
+ installable.flakeRef,
+ resolvedRef,
+ attrPath,
+ };
+
+ pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
+ }
+ }
+
+ store->buildPaths(pathsToBuild);
+
+ updateProfile(manifest.build(store));
+ }
+};
+
+struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
+{
+ std::string description() override
+ {
+ return "list installed packages";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To show what packages are installed in the default profile:",
+ "nix profile info"
+ },
+ };
+ }
+
+ void run(ref<Store> store) override
+ {
+ ProfileManifest manifest(*getEvalState(), *profile);
+
+ for (size_t i = 0; i < manifest.elements.size(); ++i) {
+ auto & element(manifest.elements[i]);
+ logger->stdout("%d %s %s %s", i,
+ element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-",
+ element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-",
+ concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
+ }
+ }
+};
+
+struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
+{
+ std::string description() override
+ {
+ return "show the closure difference between each generation of a profile";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To show what changed between each generation of the NixOS system profile:",
+ "nix profile diff-closure --profile /nix/var/nix/profiles/system"
+ },
+ };
+ }
+
+ void run(ref<Store> store) override
+ {
+ auto [gens, curGen] = findGenerations(*profile);
+
+ std::optional<Generation> prevGen;
+ bool first = true;
+
+ for (auto & gen : gens) {
+ if (prevGen) {
+ if (!first) std::cout << "\n";
+ first = false;
+ std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number);
+ printClosureDiff(store,
+ store->followLinksToStorePath(prevGen->path),
+ store->followLinksToStorePath(gen.path),
+ " ");
+ }
+
+ prevGen = gen;
+ }
+ }
+};
+
+struct CmdProfile : virtual MultiCommand, virtual Command
+{
+ CmdProfile()
+ : MultiCommand({
+ {"install", []() { return make_ref<CmdProfileInstall>(); }},
+ {"remove", []() { return make_ref<CmdProfileRemove>(); }},
+ {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }},
+ {"info", []() { return make_ref<CmdProfileInfo>(); }},
+ {"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }},
+ })
+ { }
+
+ std::string description() override
+ {
+ return "manage Nix profiles";
+ }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix profile' requires a sub-command.");
+ command->second->prepare();
+ command->second->run();
+ }
+
+ void printHelp(const string & programName, std::ostream & out) override
+ {
+ MultiCommand::printHelp(programName, out);
+ }
+};
+
+static auto r1 = registerCommand<CmdProfile>("profile");
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
new file mode 100644
index 000000000..ebee4545c
--- /dev/null
+++ b/src/nix/registry.cc
@@ -0,0 +1,151 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "eval.hh"
+#include "flake/flake.hh"
+#include "store-api.hh"
+#include "fetchers.hh"
+#include "registry.hh"
+
+using namespace nix;
+using namespace nix::flake;
+
+struct CmdRegistryList : StoreCommand
+{
+ std::string description() override
+ {
+ return "list available Nix flakes";
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ using namespace fetchers;
+
+ auto registries = getRegistries(store);
+
+ for (auto & registry : registries) {
+ for (auto & entry : registry->entries) {
+ // FIXME: format nicely
+ logger->stdout("%s %s %s",
+ registry->type == Registry::Flag ? "flags " :
+ registry->type == Registry::User ? "user " :
+ registry->type == Registry::System ? "system" :
+ "global",
+ entry.from.to_string(),
+ entry.to.to_string());
+ }
+ }
+ }
+};
+
+struct CmdRegistryAdd : MixEvalArgs, Command
+{
+ std::string fromUrl, toUrl;
+
+ std::string description() override
+ {
+ return "add/replace flake in user flake registry";
+ }
+
+ CmdRegistryAdd()
+ {
+ expectArg("from-url", &fromUrl);
+ expectArg("to-url", &toUrl);
+ }
+
+ void run() override
+ {
+ auto fromRef = parseFlakeRef(fromUrl);
+ auto toRef = parseFlakeRef(toUrl);
+ fetchers::Attrs extraAttrs;
+ if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir;
+ auto userRegistry = fetchers::getUserRegistry();
+ userRegistry->remove(fromRef.input);
+ userRegistry->add(fromRef.input, toRef.input, extraAttrs);
+ userRegistry->write(fetchers::getUserRegistryPath());
+ }
+};
+
+struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
+{
+ std::string url;
+
+ std::string description() override
+ {
+ return "remove flake from user flake registry";
+ }
+
+ CmdRegistryRemove()
+ {
+ expectArg("url", &url);
+ }
+
+ void run() override
+ {
+ auto userRegistry = fetchers::getUserRegistry();
+ userRegistry->remove(parseFlakeRef(url).input);
+ userRegistry->write(fetchers::getUserRegistryPath());
+ }
+};
+
+struct CmdRegistryPin : virtual Args, EvalCommand
+{
+ std::string url;
+
+ std::string description() override
+ {
+ return "pin a flake to its current version in user flake registry";
+ }
+
+ CmdRegistryPin()
+ {
+ expectArg("url", &url);
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ auto ref = parseFlakeRef(url);
+ auto userRegistry = fetchers::getUserRegistry();
+ userRegistry->remove(ref.input);
+ auto [tree, resolved] = ref.resolve(store).input.fetch(store);
+ fetchers::Attrs extraAttrs;
+ if (ref.subdir != "") extraAttrs["dir"] = ref.subdir;
+ userRegistry->add(ref.input, resolved, extraAttrs);
+ userRegistry->write(fetchers::getUserRegistryPath());
+ }
+};
+
+struct CmdRegistry : virtual MultiCommand, virtual Command
+{
+ CmdRegistry()
+ : MultiCommand({
+ {"list", []() { return make_ref<CmdRegistryList>(); }},
+ {"add", []() { return make_ref<CmdRegistryAdd>(); }},
+ {"remove", []() { return make_ref<CmdRegistryRemove>(); }},
+ {"pin", []() { return make_ref<CmdRegistryPin>(); }},
+ })
+ {
+ }
+
+ std::string description() override
+ {
+ return "manage the flake registry";
+ }
+
+ Category category() override { return catSecondary; }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix registry' requires a sub-command.");
+ command->second->prepare();
+ command->second->run();
+ }
+
+ void printHelp(const string & programName, std::ostream & out) override
+ {
+ MultiCommand::printHelp(programName, out);
+ }
+};
+
+static auto r1 = registerCommand<CmdRegistry>("registry");
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 617d49614..fb9050d0d 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -211,12 +211,12 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
// input without clearing the input so far.
continue;
} else {
- printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+ printMsg(lvlError, e.msg());
}
} catch (Error & e) {
- printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+ printMsg(lvlError, e.msg());
} catch (Interrupted & e) {
- printMsg(lvlError, error + "%1%%2%", (settings.showTrace ? e.prefix() : ""), e.msg());
+ printMsg(lvlError, e.msg());
}
// We handled the current input fully, so we should clear it
@@ -483,10 +483,10 @@ bool NixRepl::processLine(string line)
but doing it in a child makes it easier to recover from
problems / SIGINT. */
if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
- auto drv = readDerivation(*state->store, drvPath);
+ auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputs)
- std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path));
+ std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
}
} else if (command == ":i") {
runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
@@ -760,7 +760,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs
CmdRepl()
{
- expectArgs("files", &files);
+ expectArgs({
+ .label = "files",
+ .handler = {&files},
+ .completer = completePath
+ });
}
std::string description() override
@@ -780,6 +784,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs
void run(ref<Store> store) override
{
+ evalSettings.pureEval = false;
auto repl = std::make_unique<NixRepl>(searchPath, openStore());
repl->autoArgs = getAutoArgs(*repl->state);
repl->mainLoop(files);
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 321ee1d11..cbaba9d90 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -84,31 +84,30 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
{
return {
Example{
- "To start a shell providing GNU Hello from NixOS 17.03:",
- "nix shell -f channel:nixos-17.03 hello"
+ "To start a shell providing GNU Hello from NixOS 20.03:",
+ "nix shell nixpkgs/nixos-20.03#hello"
},
Example{
"To start a shell providing youtube-dl from your 'nixpkgs' channel:",
- "nix shell nixpkgs.youtube-dl"
+ "nix shell nixpkgs#youtube-dl"
},
Example{
"To run GNU Hello:",
- "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'"
+ "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'"
},
Example{
"To run GNU Hello in a chroot store:",
- "nix shell --store ~/my-nix nixpkgs.hello -c hello"
+ "nix shell --store ~/my-nix nixpkgs#hello -c hello"
},
};
}
void run(ref<Store> store) override
{
- auto outPaths = toStorePaths(store, Build, installables);
+ auto outPaths = toStorePaths(store, Realise::Outputs, OperateOn::Output, installables);
auto accessor = store->getFSAccessor();
-
std::unordered_set<StorePath> done;
std::queue<StorePath> todo;
for (auto & path : outPaths) todo.push(path);
@@ -143,6 +142,67 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
static auto r1 = registerCommand<CmdShell>("shell");
+struct CmdRun : InstallableCommand, RunCommon
+{
+ std::vector<std::string> args;
+
+ CmdRun()
+ {
+ expectArgs({
+ .label = "args",
+ .handler = {&args},
+ .completer = completePath
+ });
+ }
+
+ std::string description() override
+ {
+ return "run a Nix application";
+ }
+
+ Examples examples() override
+ {
+ return {
+ Example{
+ "To run Blender:",
+ "nix run blender-bin"
+ },
+ };
+ }
+
+ Strings getDefaultFlakeAttrPaths() override
+ {
+ Strings res{"defaultApp." + settings.thisSystem.get()};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths())
+ res.push_back(s);
+ return res;
+ }
+
+ Strings getDefaultFlakeAttrPathPrefixes() override
+ {
+ Strings res{"apps." + settings.thisSystem.get() + ".", "packages"};
+ for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes())
+ res.push_back(s);
+ return res;
+ }
+
+ void run(ref<Store> store) override
+ {
+ auto state = getEvalState();
+
+ auto app = installable->toApp(*state);
+
+ state->store->buildPaths(app.context);
+
+ Strings allArgs{app.program};
+ for (auto & i : args) allArgs.push_back(i);
+
+ runProgram(store, app.program, allArgs);
+ }
+};
+
+static auto r2 = registerCommand<CmdRun>("run");
+
void chrootHelper(int argc, char * * argv)
{
int p = 1;
diff --git a/src/nix/search.cc b/src/nix/search.cc
index ba72c1e79..65a1e1818 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -6,8 +6,9 @@
#include "get-drvs.hh"
#include "common-args.hh"
#include "json.hh"
-#include "json-to-value.hh"
#include "shared.hh"
+#include "eval-cache.hh"
+#include "attr-path.hh"
#include <regex>
#include <fstream>
@@ -25,33 +26,17 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos
m.empty()
? s
: std::string(m.prefix())
- + ANSI_RED + std::string(m.str()) + postfix
+ + ANSI_GREEN + std::string(m.str()) + postfix
+ std::string(m.suffix());
}
-struct CmdSearch : SourceExprCommand, MixJSON
+struct CmdSearch : InstallableCommand, MixJSON
{
std::vector<std::string> res;
- bool writeCache = true;
- bool useCache = true;
-
CmdSearch()
{
expectArgs("regex", &res);
-
- addFlag({
- .longName = "update-cache",
- .shortName = 'u',
- .description = "update the package search cache",
- .handler = {[&]() { writeCache = true; useCache = false; }}
- });
-
- addFlag({
- .longName = "no-cache",
- .description = "do not use or update the package search cache",
- .handler = {[&]() { writeCache = false; useCache = false; }}
- });
}
std::string description() override
@@ -63,24 +48,32 @@ struct CmdSearch : SourceExprCommand, MixJSON
{
return {
Example{
- "To show all available packages:",
+ "To show all packages in the flake in the current directory:",
"nix search"
},
Example{
- "To show any packages containing 'blender' in its name or description:",
- "nix search blender"
+ "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:",
+ "nix search nixpkgs blender"
},
Example{
"To search for Firefox or Chromium:",
- "nix search 'firefox|chromium'"
+ "nix search nixpkgs 'firefox|chromium'"
},
Example{
- "To search for git and frontend or gui:",
- "nix search git 'frontend|gui'"
+ "To search for packages containing 'git' and either 'frontend' or 'gui':",
+ "nix search nixpkgs git 'frontend|gui'"
}
};
}
+ Strings getDefaultFlakeAttrPaths() override
+ {
+ return {
+ "packages." + settings.thisSystem.get() + ".",
+ "legacyPackages." + settings.thisSystem.get() + "."
+ };
+ }
+
void run(ref<Store> store) override
{
settings.readOnlyMode = true;
@@ -88,189 +81,107 @@ struct CmdSearch : SourceExprCommand, MixJSON
// Empty search string should match all packages
// Use "^" here instead of ".*" due to differences in resulting highlighting
// (see #1893 -- libc++ claims empty search string is not in POSIX grammar)
- if (res.empty()) {
+ if (res.empty())
res.push_back("^");
- }
std::vector<std::regex> regexes;
regexes.reserve(res.size());
- for (auto &re : res) {
+ for (auto & re : res)
regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase));
- }
auto state = getEvalState();
auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
- auto sToplevel = state->symbols.create("_toplevel");
- auto sRecurse = state->symbols.create("recurseForDerivations");
-
- bool fromCache = false;
+ uint64_t results = 0;
- std::map<std::string, std::string> results;
-
- std::function<void(Value *, std::string, bool, JSONObject *)> doExpr;
-
- doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) {
- debug("at attribute '%s'", attrPath);
+ std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)> visit;
+ visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)
+ {
+ Activity act(*logger, lvlInfo, actUnknown,
+ fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
try {
- uint found = 0;
+ auto recurse = [&]()
+ {
+ for (const auto & attr : cursor.getAttrs()) {
+ auto cursor2 = cursor.getAttr(attr);
+ auto attrPath2(attrPath);
+ attrPath2.push_back(attr);
+ visit(*cursor2, attrPath2);
+ }
+ };
- state->forceValue(*v);
+ if (cursor.isDerivation()) {
+ size_t found = 0;
- if (v->type == tLambda && toplevel) {
- Value * v2 = state->allocValue();
- state->autoCallFunction(*state->allocBindings(1), *v, *v2);
- v = v2;
- state->forceValue(*v);
- }
+ DrvName name(cursor.getAttr("name")->getString());
- if (state->isDerivation(*v)) {
+ auto aMeta = cursor.maybeGetAttr("meta");
+ auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr;
+ auto description = aDescription ? aDescription->getString() : "";
+ std::replace(description.begin(), description.end(), '\n', ' ');
+ auto attrPath2 = concatStringsSep(".", attrPath);
- DrvInfo drv(*state, attrPath, v->attrs);
- std::string description;
std::smatch attrPathMatch;
std::smatch descriptionMatch;
std::smatch nameMatch;
- std::string name;
-
- DrvName parsed(drv.queryName());
-
- for (auto &regex : regexes) {
- std::regex_search(attrPath, attrPathMatch, regex);
- name = parsed.name;
- std::regex_search(name, nameMatch, regex);
-
- description = drv.queryMetaString("description");
- std::replace(description.begin(), description.end(), '\n', ' ');
+ for (auto & regex : regexes) {
+ std::regex_search(attrPath2, attrPathMatch, regex);
+ std::regex_search(name.name, nameMatch, regex);
std::regex_search(description, descriptionMatch, regex);
-
if (!attrPathMatch.empty()
|| !nameMatch.empty()
|| !descriptionMatch.empty())
- {
found++;
- }
}
if (found == res.size()) {
+ results++;
if (json) {
-
- auto jsonElem = jsonOut->object(attrPath);
-
- jsonElem.attr("pkgName", parsed.name);
- jsonElem.attr("version", parsed.version);
+ auto jsonElem = jsonOut->object(attrPath2);
+ jsonElem.attr("pname", name.name);
+ jsonElem.attr("version", name.version);
jsonElem.attr("description", description);
-
} else {
- auto name = hilite(parsed.name, nameMatch, "\e[0;2m")
- + std::string(parsed.fullName, parsed.name.length());
- results[attrPath] = fmt(
- "* %s (%s)\n %s\n",
- wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")),
- wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")),
- hilite(description, descriptionMatch, ANSI_NORMAL));
- }
- }
-
- if (cache) {
- cache->attr("type", "derivation");
- cache->attr("name", drv.queryName());
- cache->attr("system", drv.querySystem());
- if (description != "") {
- auto meta(cache->object("meta"));
- meta.attr("description", description);
+ auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
+ if (results > 1) logger->stdout("");
+ logger->stdout(
+ "* %s%s",
+ wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
+ name.version != "" ? " (" + name.version + ")" : "");
+ if (description != "")
+ logger->stdout(
+ " %s", hilite(description, descriptionMatch, ANSI_NORMAL));
}
}
}
- else if (v->type == tAttrs) {
+ else if (
+ attrPath.size() == 0
+ || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2)
+ || (attrPath[0] == "packages" && attrPath.size() <= 2))
+ recurse();
- if (!toplevel) {
- auto attrs = v->attrs;
- Bindings::iterator j = attrs->find(sRecurse);
- if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) {
- debug("skip attribute '%s'", attrPath);
- return;
- }
- }
-
- bool toplevel2 = false;
- if (!fromCache) {
- Bindings::iterator j = v->attrs->find(sToplevel);
- toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos);
- }
-
- for (auto & i : *v->attrs) {
- auto cache2 =
- cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr;
- doExpr(i.value,
- attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name,
- toplevel2 || fromCache, cache2 ? cache2.get() : nullptr);
- }
+ else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
+ auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
+ if (attr && attr->getBool())
+ recurse();
}
- } catch (AssertionError & e) {
- } catch (Error & e) {
- if (!toplevel) {
- e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath));
+ } catch (EvalError & e) {
+ if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages"))
throw;
- }
}
};
- Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json";
-
- if (useCache && pathExists(jsonCacheFileName)) {
-
- warn("using cached results; pass '-u' to update the cache");
-
- Value vRoot;
- parseJSON(*state, readFile(jsonCacheFileName), vRoot);
-
- fromCache = true;
-
- doExpr(&vRoot, "", true, nullptr);
- }
+ for (auto & [cursor, prefix] : installable->getCursors(*state, true))
+ visit(*cursor, parseAttrPath(*state, prefix));
- else {
- createDirs(dirOf(jsonCacheFileName));
-
- Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid());
-
- std::ofstream jsonCacheFile;
-
- try {
- // iostream considered harmful
- jsonCacheFile.exceptions(std::ofstream::failbit);
- jsonCacheFile.open(tmpFile);
-
- auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr;
-
- doExpr(getSourceExpr(*state), "", true, cache.get());
-
- } catch (std::exception &) {
- /* Fun fact: catching std::ios::failure does not work
- due to C++11 ABI shenanigans.
- https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */
- if (!jsonCacheFile)
- throw Error("error writing to %s", tmpFile);
- throw;
- }
-
- if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1)
- throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
- }
-
- if (!json && results.size() == 0)
+ if (!json && !results)
throw Error("no results for the given search term(s)!");
-
- RunPager pager;
- for (auto el : results) std::cout << el.second << "\n";
-
}
};
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 5d77cfdca..9fd26e2d7 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -33,7 +33,7 @@ struct CmdShowDerivation : InstallablesCommand
return {
Example{
"To show the store derivation that results from evaluating the Hello package:",
- "nix show-derivation nixpkgs.hello"
+ "nix show-derivation nixpkgs#hello"
},
Example{
"To show the full derivation graph (if available) that produced your NixOS system:",
@@ -69,10 +69,10 @@ struct CmdShowDerivation : InstallablesCommand
auto outputsObj(drvObj.object("outputs"));
for (auto & output : drv.outputs) {
auto outputObj(outputsObj.object(output.first));
- outputObj.attr("path", store->printStorePath(output.second.path));
- if (output.second.hash) {
- outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo());
- outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false));
+ outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
+ if (auto hash = std::get_if<DerivationOutputFixed>(&output.second.output)) {
+ outputObj.attr("hashAlgo", hash->hash.printMethodAlgo());
+ outputObj.attr("hash", hash->hash.hash.to_string(Base16, false));
}
}
}
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 6c9b9a792..7821a5432 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -105,7 +105,8 @@ struct CmdSignPaths : StorePathsCommand
.shortName = 'k',
.description = "file containing the secret signing key",
.labels = {"file"},
- .handler = {&secretKeyFile}
+ .handler = {&secretKeyFile},
+ .completer = completePath
});
}
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index bb5e4529b..fc7a9765c 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -77,26 +77,29 @@ struct CmdVerify : StorePathsCommand
try {
checkInterrupt();
- Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath));
-
MaintainCount<std::atomic<size_t>> mcActive(active);
update();
auto info = store->queryPathInfo(store->parseStorePath(storePath));
+ // Note: info->path can be different from storePath
+ // for binary cache stores when using --all (since we
+ // can't enumerate names efficiently).
+ Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path)));
+
if (!noContents) {
std::unique_ptr<AbstractHashSink> hashSink;
if (!info->ca)
- hashSink = std::make_unique<HashSink>(*info->narHash.type);
+ hashSink = std::make_unique<HashSink>(info->narHash->type);
else
- hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
+ hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
store->narFromPath(info->path, *hashSink);
auto hash = hashSink->finish();
- if (hash.first != info->narHash) {
+ if (hash.first != *info->narHash) {
corrupted++;
act2.result(resCorruptedPath, store->printStorePath(info->path));
logError({
@@ -104,7 +107,7 @@ struct CmdVerify : StorePathsCommand
.hint = hintfmt(
"path '%s' was modified! expected hash '%s', got '%s'",
store->printStorePath(info->path),
- info->narHash.to_string(Base32, true),
+ info->narHash->to_string(Base32, true),
hash.first.to_string(Base32, true))
});
}
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 167c974ee..7e630b745 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -55,15 +55,15 @@ struct CmdWhyDepends : SourceExprCommand
return {
Example{
"To show one path through the dependency graph leading from Hello to Glibc:",
- "nix why-depends nixpkgs.hello nixpkgs.glibc"
+ "nix why-depends nixpkgs#hello nixpkgs#glibc"
},
Example{
"To show all files and paths in the dependency graph leading from Thunderbird to libX11:",
- "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11"
+ "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11"
},
Example{
"To show why Glibc depends on itself:",
- "nix why-depends nixpkgs.glibc nixpkgs.glibc"
+ "nix why-depends nixpkgs#glibc nixpkgs#glibc"
},
};
}
@@ -72,17 +72,19 @@ struct CmdWhyDepends : SourceExprCommand
void run(ref<Store> store) override
{
- auto package = parseInstallable(*this, store, _package, false);
- auto packagePath = toStorePath(store, Build, package);
- auto dependency = parseInstallable(*this, store, _dependency, false);
- auto dependencyPath = toStorePath(store, NoBuild, dependency);
+ auto package = parseInstallable(store, _package);
+ auto packagePath = toStorePath(store, Realise::Outputs, operateOn, package);
+ auto dependency = parseInstallable(store, _dependency);
+ auto dependencyPath = toStorePath(store, Realise::Derivation, operateOn, dependency);
auto dependencyPathHash = dependencyPath.hashPart();
StorePathSet closure;
store->computeFSClosure({packagePath}, closure, false, false);
if (!closure.count(dependencyPath)) {
- printError("'%s' does not depend on '%s'", package->what(), dependency->what());
+ printError("'%s' does not depend on '%s'",
+ store->printStorePath(packagePath),
+ store->printStorePath(dependencyPath));
return;
}
@@ -106,7 +108,11 @@ struct CmdWhyDepends : SourceExprCommand
std::map<StorePath, Node> graph;
for (auto & path : closure)
- graph.emplace(path, Node { .path = path, .refs = store->queryPathInfo(path)->references });
+ graph.emplace(path, Node {
+ .path = path,
+ .refs = store->queryPathInfo(path)->references,
+ .dist = path == dependencyPath ? 0 : inf
+ });
// Transpose the graph.
for (auto & node : graph)
@@ -115,8 +121,6 @@ struct CmdWhyDepends : SourceExprCommand
/* Run Dijkstra's shortest path algorithm to get the distance
of every path in the closure to 'dependency'. */
- graph.emplace(dependencyPath, Node { .path = dependencyPath, .dist = 0 });
-
std::priority_queue<Node *> queue;
queue.push(&graph.at(dependencyPath));
diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 17b63d978..40f1a4f76 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -182,3 +182,56 @@ clearCacheCache
nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey"
fi # HAVE_LIBSODIUM
+
+
+unset _NIX_FORCE_HTTP
+
+
+# Test 'nix verify --all' on a binary cache.
+nix verify -vvvvv --all --store file://$cacheDir --no-trust
+
+
+# Test local NAR caching.
+narCache=$TEST_ROOT/nar-cache
+rm -rf $narCache
+mkdir $narCache
+
+[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
+
+rm -rfv "$cacheDir/nar"
+
+[[ $(nix cat-store --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]]
+
+(! nix cat-store --store file://$cacheDir $outPath/foobar)
+
+
+# Test NAR listing generation.
+clearCache
+
+outPath=$(nix-build --no-out-link -E '
+ with import ./config.nix;
+ mkDerivation {
+ name = "nar-listing";
+ buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link";
+ }
+')
+
+nix copy --to file://$cacheDir?write-nar-listing=1 $outPath
+
+[[ $(cat $cacheDir/$(basename $outPath).ls) = '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' ]]
+
+
+# Test debug info index generation.
+clearCache
+
+outPath=$(nix-build --no-out-link -E '
+ with import ./config.nix;
+ mkDerivation {
+ name = "debug-info";
+ buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug";
+ }
+')
+
+nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath
+
+[[ $(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) = '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' ]]
diff --git a/tests/check.sh b/tests/check.sh
index 5f25d04cb..5f4997e28 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -61,30 +61,30 @@ nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/l
[ "$status" = "1" ]
grep 'differs from previous round' $TEST_ROOT/log
-path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '')
+path=$(nix-build check.nix -A fetchurl --no-out-link)
chmod +w $path
echo foo > $path
chmod -w $path
-nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --check
# Note: "check" doesn't repair anything, it just compares to the hash stored in the database.
[[ $(cat $path) = foo ]]
-nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors ''
+nix-build check.nix -A fetchurl --no-out-link --repair
[[ $(cat $path) != foo ]]
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link || status=$?
[ "$status" = "102" ]
echo -n > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors ''
+nix-build check.nix -A hashmismatch --no-out-link
echo 'Hello World' > ./dummy
-nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$?
+nix-build check.nix -A hashmismatch --no-out-link --check || status=$?
[ "$status" = "102" ]
# Multiple failures with --keep-going
nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$?
+nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going || status=$?
[ "$status" = "110" ]
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 73fe77345..308126094 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -1,6 +1,6 @@
set -e
-export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)
+export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
export NIX_STORE_DIR
if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
# Maybe the build directory is symlinked.
@@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
export NIX_STATE_DIR=$TEST_ROOT/var/nix
export NIX_CONF_DIR=$TEST_ROOT/etc
+export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/daemon-socket
unset NIX_USER_CONF_FILES
export _NIX_TEST_SHARED=$TEST_ROOT/shared
if [[ -n $NIX_STORE ]]; then
@@ -76,7 +77,7 @@ startDaemon() {
rm -f $NIX_STATE_DIR/daemon-socket/socket
nix-daemon &
for ((i = 0; i < 30; i++)); do
- if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi
+ if [ -e $NIX_DAEMON_SOCKET_PATH ]; then break; fi
sleep 1
done
pidDaemon=$!
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index d9c9874f5..cedd796f7 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -31,44 +31,46 @@ rev2=$(git -C $repo rev-parse HEAD)
# Fetch a worktree
unset _NIX_FORCE_HTTP
-path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath")
+[[ $path0 = $path0_ ]]
export _NIX_FORCE_HTTP=1
[[ $(tail -n 1 $path0/hello) = "hello" ]]
# Fetch the default branch.
-path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $(cat $path/hello) = world ]]
# In pure eval mode, fetchGit without a revision should fail.
-[[ $(nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]]
-(! nix eval --pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))")
+[[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")") = world ]]
+(! nix eval --raw --expr "builtins.readFile (fetchGit file://$repo + \"/hello\")")
# Fetch using an explicit revision hash.
-path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
+path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
# In pure eval mode, fetchGit with a revision should succeed.
-[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]]
+[[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]]
# Fetch again. This should be cached.
mv $repo ${repo}-tmp
-path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $path = $path2 ]]
-[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]]
-[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
+[[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
# Fetching with a explicit hash should succeed.
-path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
-path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath")
[[ $(cat $path2/hello) = utrecht ]]
mv ${repo}-tmp $repo
# Using a clean working tree should produce the same result.
-path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $path = $path2 ]]
# Using an unclean tree should yield the tracked but uncommitted changes.
@@ -80,59 +82,65 @@ git -C $repo add dir1/foo
git -C $repo rm hello
unset _NIX_FORCE_HTTP
-path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[ ! -e $path2/hello ]
[ ! -e $path2/bar ]
[ ! -e $path2/dir2/bar ]
[ ! -e $path2/.git ]
[[ $(cat $path2/dir1/foo) = foo ]]
-[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
# ... unless we're using an explicit ref or rev.
-path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
+path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
[[ $path = $path3 ]]
-path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath")
+path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path3 ]]
# Committing should not affect the store path.
git -C $repo commit -m 'Bla3' -a
-path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath")
+path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath")
[[ $path2 = $path4 ]]
+nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$?
+[[ "$status" = "102" ]]
+
+path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath")
+[[ $path = $path5 ]]
+
# tarball-ttl should be ignored if we specify a rev
echo delft > $repo/hello
git -C $repo add hello
git -C $repo commit -m 'Bla4'
rev3=$(git -C $repo rev-parse HEAD)
-nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null
+nix eval --tarball-ttl 3600 --expr "builtins.fetchGit { url = $repo; rev = \"$rev3\"; }" >/dev/null
# Update 'path' to reflect latest master
-path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath")
# Check behavior when non-master branch is used
git -C $repo checkout $rev2 -b dev
echo dev > $repo/hello
# File URI uses dirty tree unless specified otherwise
-path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath")
[ $(cat $path2/hello) = dev ]
# Using local path with branch other than 'master' should work when clean or dirty
-path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
# (check dirty-tree handling was used)
-[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]]
# Committing shouldn't change store path, or switch to using 'master'
git -C $repo commit -m 'Bla5' -a
-path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
+path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath")
[[ $(cat $path4/hello) = dev ]]
[[ $path3 = $path4 ]]
# Confirm same as 'dev' branch
-path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
+path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
[[ $path3 = $path5 ]]
@@ -141,19 +149,18 @@ rm -rf $TEST_HOME/.cache/nix
# Try again, but without 'git' on PATH. This should fail.
NIX=$(command -v nix)
-# This should fail
-(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
+(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
# Try again, with 'git' available. This should work.
-path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
+path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
[[ $path3 = $path5 ]]
# Fetching a shallow repo shouldn't work by default, because we can't
# return a revCount.
git clone --depth 1 file://$repo $TEST_ROOT/shallow
-(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath")
+(! nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath")
# But you can request a shallow clone, which won't return a revCount.
-path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
+path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
[[ $path3 = $path6 ]]
-[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
+[[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh
index 23934698e..52926040b 100644
--- a/tests/fetchGitRefs.sh
+++ b/tests/fetchGitRefs.sh
@@ -19,7 +19,7 @@ echo utrecht > "$repo"/hello
git -C "$repo" add hello
git -C "$repo" commit -m 'Bla1'
-path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
+path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
# Test various combinations of ref names
# (taken from the git project)
@@ -42,7 +42,7 @@ valid_ref() {
{ set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
git check-ref-format --branch "$1" >/dev/null
git -C "$repo" branch "$1" master >/dev/null
- path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath")
+ path1=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath")
[[ $path1 = $path ]]
git -C "$repo" branch -D "$1" >/dev/null
}
@@ -56,7 +56,7 @@ invalid_ref() {
else
(! git check-ref-format --branch "$1" >/dev/null 2>&1)
fi
- nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null
+ nix --debug eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null
}
diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
index 4c2c13f1a..5f104355f 100644
--- a/tests/fetchGitSubmodules.sh
+++ b/tests/fetchGitSubmodules.sh
@@ -38,18 +38,18 @@ git -C $rootRepo commit -m "Add submodule"
rev=$(git -C $rootRepo rev-parse HEAD)
-r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
-r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
-r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
+r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
[[ $r1 == $r2 ]]
[[ $r2 != $r3 ]]
-r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath")
-r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath")
-r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
-r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
-r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath")
+r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath")
+r6=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
[[ $r1 == $r4 ]]
[[ $r4 == $r5 ]]
@@ -57,19 +57,19 @@ r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submo
[[ $r6 == $r7 ]]
[[ $r7 == $r8 ]]
-have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
+have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
[[ $have_submodules == false ]]
-have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
+have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
[[ $have_submodules == false ]]
-have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
+have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
[[ $have_submodules == true ]]
-pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
-pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
-pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
-pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithoutSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+pathWithSubmodules=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
# The resulting store path cannot be the same.
[[ $pathWithoutSubmodules != $pathWithSubmodules ]]
@@ -91,7 +91,7 @@ test "$(find "$pathWithSubmodules" -name .git)" = ""
# Git repos without submodules can be fetched with submodules = true.
subRev=$(git -C $subRepo rev-parse HEAD)
-noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
-noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
+noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
+noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]
diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh
index 4088dbd39..af8ef8d5b 100644
--- a/tests/fetchMercurial.sh
+++ b/tests/fetchMercurial.sh
@@ -9,7 +9,7 @@ clearStore
repo=$TEST_ROOT/hg
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix
hg init $repo
echo '[ui]' >> $repo/.hg/hgrc
@@ -26,43 +26,43 @@ hg commit --cwd $repo -m 'Bla2'
rev2=$(hg log --cwd $repo -r tip --template '{node}')
# Fetch the default branch.
-path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
+path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
[[ $(cat $path/hello) = world ]]
# In pure eval mode, fetchGit without a revision should fail.
-[[ $(nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]]
-(! nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))")
+[[ $(nix eval --impure --raw --expr "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]]
+(! nix eval --raw --expr "builtins.readFile (fetchMercurial file://$repo + \"/hello\")")
# Fetch using an explicit revision hash.
-path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
# In pure eval mode, fetchGit with a revision should succeed.
-[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]]
+[[ $(nix eval --raw --expr "builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]]
# Fetch again. This should be cached.
mv $repo ${repo}-tmp
-path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
[[ $path = $path2 ]]
-[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]]
-[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]]
-[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).branch") = default ]]
+[[ $(nix eval --impure --expr "(builtins.fetchMercurial file://$repo).revCount") = 1 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]]
# But with TTL 0, it should fail.
-(! nix eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)")
+(! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo")
# Fetching with a explicit hash should succeed.
-path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath")
[[ $path = $path2 ]]
-path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
+path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath")
[[ $(cat $path2/hello) = utrecht ]]
mv ${repo}-tmp $repo
# Using a clean working tree should produce the same result.
-path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
[[ $path = $path2 ]]
# Using an unclean tree should yield the tracked but uncommitted changes.
@@ -73,21 +73,21 @@ echo bar > $repo/dir2/bar
hg add --cwd $repo dir1/foo
hg rm --cwd $repo hello
-path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath")
+path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath")
[ ! -e $path2/hello ]
[ ! -e $path2/bar ]
[ ! -e $path2/dir2/bar ]
[ ! -e $path2/.hg ]
[[ $(cat $path2/dir1/foo) = foo ]]
-[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
+[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]]
-# ... unless we're using an explicit rev.
-path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath")
+# ... unless we're using an explicit ref.
+path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath")
[[ $path = $path3 ]]
# Committing should not affect the store path.
hg commit --cwd $repo -m 'Bla3'
-path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath")
+path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath")
[[ $path2 = $path4 ]]
diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh
index 2535651b0..0f2044342 100644
--- a/tests/fetchurl.sh
+++ b/tests/fetchurl.sh
@@ -5,7 +5,7 @@ clearStore
# Test fetching a flat file.
hash=$(nix-hash --flat --type sha256 ./fetchurl.sh)
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -14,7 +14,7 @@ clearStore
hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link)
cmp $outPath fetchurl.sh
@@ -25,26 +25,24 @@ hash=$(nix hash-file ./fetchurl.sh)
[[ $hash =~ ^sha256- ]]
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '')
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link)
cmp $outPath fetchurl.sh
-# Test the hashed mirror feature.
+# Test that we can substitute from a different store dir.
clearStore
-hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh)
-hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh)
+other_store=file://$TEST_ROOT/other_store?store=/fnord/store
+
+hash=$(nix hash-file --type sha256 --base16 ./fetchurl.sh)
-mirror=$TEST_ROOT/hashed-mirror
-rm -rf $mirror
-mkdir -p $mirror/sha512
-ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32
+storePath=$(nix --store $other_store add-to-store --flat ./fetchurl.sh)
-outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror")
+outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store)
# Test hashed mirrors with an SRI hash.
-nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \
- --argstr name bla --no-out-link --hashed-mirrors "file://$mirror"
+nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha256 $hash) \
+ --no-out-link --substituters $other_store
# Test unpacking a NAR.
rm -rf $TEST_ROOT/archive
diff --git a/tests/filter-source.sh b/tests/filter-source.sh
index 1f8dceee5..ba34d2eac 100644
--- a/tests/filter-source.sh
+++ b/tests/filter-source.sh
@@ -10,10 +10,16 @@ touch $TEST_ROOT/filterin/bak
touch $TEST_ROOT/filterin/bla.c.bak
ln -s xyzzy $TEST_ROOT/filterin/link
-nix-build ./filter-source.nix -o $TEST_ROOT/filterout
+checkFilter() {
+ test ! -e $1/foo/bar
+ test -e $1/xyzzy
+ test -e $1/bak
+ test ! -e $1/bla.c.bak
+ test ! -L $1/link
+}
-test ! -e $TEST_ROOT/filterout/foo/bar
-test -e $TEST_ROOT/filterout/xyzzy
-test -e $TEST_ROOT/filterout/bak
-test ! -e $TEST_ROOT/filterout/bla.c.bak
-test ! -L $TEST_ROOT/filterout/link
+nix-build ./filter-source.nix -o $TEST_ROOT/filterout1
+checkFilter $TEST_ROOT/filterout1
+
+nix-build ./path.nix -o $TEST_ROOT/filterout2
+checkFilter $TEST_ROOT/filterout2
diff --git a/tests/flakes.sh b/tests/flakes.sh
new file mode 100644
index 000000000..5aec563ac
--- /dev/null
+++ b/tests/flakes.sh
@@ -0,0 +1,718 @@
+source common.sh
+
+if [[ -z $(type -p git) ]]; then
+ echo "Git not installed; skipping flake tests"
+ exit 99
+fi
+
+if [[ -z $(type -p hg) ]]; then
+ echo "Mercurial not installed; skipping flake tests"
+ exit 99
+fi
+
+clearStore
+rm -rf $TEST_HOME/.cache $TEST_HOME/.config
+
+registry=$TEST_ROOT/registry.json
+
+flake1Dir=$TEST_ROOT/flake1
+flake2Dir=$TEST_ROOT/flake2
+flake3Dir=$TEST_ROOT/flake3
+flake5Dir=$TEST_ROOT/flake5
+flake6Dir=$TEST_ROOT/flake6
+flake7Dir=$TEST_ROOT/flake7
+templatesDir=$TEST_ROOT/templates
+nonFlakeDir=$TEST_ROOT/nonFlake
+flakeA=$TEST_ROOT/flakeA
+flakeB=$TEST_ROOT/flakeB
+
+for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $templatesDir $nonFlakeDir $flakeA $flakeB; do
+ rm -rf $repo $repo.tmp
+ mkdir $repo
+ git -C $repo init
+ git -C $repo config user.email "foobar@example.com"
+ git -C $repo config user.name "Foobar"
+done
+
+cat > $flake1Dir/flake.nix <<EOF
+{
+ description = "Bla bla";
+
+ outputs = inputs: rec {
+ packages.$system.foo = import ./simple.nix;
+ defaultPackage.$system = packages.$system.foo;
+
+ # To test "nix flake init".
+ legacyPackages.x86_64-linux.hello = import ./simple.nix;
+ };
+}
+EOF
+
+cp ./simple.nix ./simple.builder.sh ./config.nix $flake1Dir/
+git -C $flake1Dir add flake.nix simple.nix simple.builder.sh config.nix
+git -C $flake1Dir commit -m 'Initial'
+
+cat > $flake2Dir/flake.nix <<EOF
+{
+ description = "Fnord";
+
+ outputs = { self, flake1 }: rec {
+ packages.$system.bar = flake1.packages.$system.foo;
+ };
+}
+EOF
+
+git -C $flake2Dir add flake.nix
+git -C $flake2Dir commit -m 'Initial'
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ description = "Fnord";
+
+ outputs = { self, flake2 }: rec {
+ packages.$system.xyzzy = flake2.packages.$system.bar;
+
+ checks = {
+ xyzzy = packages.$system.xyzzy;
+ };
+ };
+}
+EOF
+
+git -C $flake3Dir add flake.nix
+git -C $flake3Dir commit -m 'Initial'
+
+cat > $nonFlakeDir/README.md <<EOF
+FNORD
+EOF
+
+git -C $nonFlakeDir add README.md
+git -C $nonFlakeDir commit -m 'Initial'
+
+cat > $registry <<EOF
+{
+ "version": 2,
+ "flakes": [
+ { "from": {
+ "type": "indirect",
+ "id": "flake1"
+ },
+ "to": {
+ "type": "git",
+ "url": "file://$flake1Dir"
+ }
+ },
+ { "from": {
+ "type": "indirect",
+ "id": "flake2"
+ },
+ "to": {
+ "type": "git",
+ "url": "file://$flake2Dir"
+ }
+ },
+ { "from": {
+ "type": "indirect",
+ "id": "flake3"
+ },
+ "to": {
+ "type": "git",
+ "url": "file://$flake3Dir"
+ }
+ },
+ { "from": {
+ "type": "indirect",
+ "id": "flake4"
+ },
+ "to": {
+ "type": "indirect",
+ "id": "flake3"
+ }
+ },
+ { "from": {
+ "type": "indirect",
+ "id": "flake5"
+ },
+ "to": {
+ "type": "hg",
+ "url": "file://$flake5Dir"
+ }
+ },
+ { "from": {
+ "type": "indirect",
+ "id": "nixpkgs"
+ },
+ "to": {
+ "type": "indirect",
+ "id": "flake1"
+ }
+ },
+ { "from": {
+ "type": "indirect",
+ "id": "templates"
+ },
+ "to": {
+ "type": "git",
+ "url": "file://$templatesDir"
+ }
+ }
+ ]
+}
+EOF
+
+# Test 'nix flake list'.
+[[ $(nix registry list | wc -l) == 7 ]]
+
+# Test 'nix flake info'.
+nix flake info flake1 | grep -q 'URL: .*flake1.*'
+
+# Test 'nix flake info' on a local flake.
+(cd $flake1Dir && nix flake info) | grep -q 'URL: .*flake1.*'
+(cd $flake1Dir && nix flake info .) | grep -q 'URL: .*flake1.*'
+nix flake info $flake1Dir | grep -q 'URL: .*flake1.*'
+
+# Test 'nix flake info --json'.
+json=$(nix flake info flake1 --json | jq .)
+[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]]
+[[ -d $(echo "$json" | jq -r .path) ]]
+[[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]]
+hash1=$(echo "$json" | jq -r .revision)
+
+echo -n '# foo' >> $flake1Dir/flake.nix
+git -C $flake1Dir commit -a -m 'Foo'
+hash2=$(nix flake info flake1 --json --refresh | jq -r .revision)
+[[ $hash1 != $hash2 ]]
+
+# Test 'nix build' on a flake.
+nix build -o $TEST_ROOT/result flake1#foo
+[[ -e $TEST_ROOT/result/hello ]]
+
+# Test defaultPackage.
+nix build -o $TEST_ROOT/result flake1
+[[ -e $TEST_ROOT/result/hello ]]
+
+nix build -o $TEST_ROOT/result $flake1Dir
+nix build -o $TEST_ROOT/result git+file://$flake1Dir
+
+# Check that store symlinks inside a flake are not interpreted as flakes.
+nix build -o $flake1Dir/result git+file://$flake1Dir
+nix path-info $flake1Dir/result
+
+# 'getFlake' on a mutable flakeref should fail in pure mode, but succeed in impure mode.
+(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").defaultPackage.$system")
+nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").defaultPackage.$system" --impure
+
+# 'getFlake' on an immutable flakeref should succeed even in pure mode.
+nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").defaultPackage.$system"
+
+# Building a flake with an unlocked dependency should fail in pure mode.
+(! nix build -o $TEST_ROOT/result flake2#bar --no-registries)
+(! nix eval --expr "builtins.getFlake \"$flake2Dir\"")
+
+# But should succeed in impure mode.
+(! nix build -o $TEST_ROOT/result flake2#bar --impure)
+nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file
+
+# Building a local flake with an unlocked dependency should fail with --no-update-lock-file.
+nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes'
+
+# But it should succeed without that flag.
+nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file
+nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes'
+nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file
+[[ -e $flake2Dir/flake.lock ]]
+[[ -z $(git -C $flake2Dir diff master) ]]
+
+# Rerunning the build should not change the lockfile.
+nix build -o $TEST_ROOT/result $flake2Dir#bar
+[[ -z $(git -C $flake2Dir diff master) ]]
+
+# Building with a lockfile should not require a fetch of the registry.
+nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh
+nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh
+
+# Updating the flake should not change the lockfile.
+nix flake update $flake2Dir
+[[ -z $(git -C $flake2Dir diff master) ]]
+
+# Now we should be able to build the flake in pure mode.
+nix build -o $TEST_ROOT/result flake2#bar
+
+# Or without a registry.
+nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh
+
+# Test whether indirect dependencies work.
+nix build -o $TEST_ROOT/result $flake3Dir#xyzzy
+git -C $flake3Dir add flake.lock
+
+# Add dependency to flake3.
+rm $flake3Dir/flake.nix
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ description = "Fnord";
+
+ outputs = { self, flake1, flake2 }: rec {
+ packages.$system.xyzzy = flake2.packages.$system.bar;
+ packages.$system."sth sth" = flake1.packages.$system.foo;
+ };
+}
+EOF
+
+git -C $flake3Dir add flake.nix
+git -C $flake3Dir commit -m 'Update flake.nix'
+
+# Check whether `nix build` works with an incomplete lockfile
+nix build -o $TEST_ROOT/result $flake3Dir#"sth sth"
+nix build -o $TEST_ROOT/result $flake3Dir#"sth%20sth"
+
+# Check whether it saved the lockfile
+(! [[ -z $(git -C $flake3Dir diff master) ]])
+
+git -C $flake3Dir add flake.lock
+
+git -C $flake3Dir commit -m 'Add lockfile'
+
+# Test whether registry caching works.
+nix registry list --flake-registry file://$registry | grep -q flake3
+mv $registry $registry.tmp
+nix-store --gc
+nix registry list --flake-registry file://$registry --refresh | grep -q flake3
+mv $registry.tmp $registry
+
+# Test whether flakes are registered as GC roots for offline use.
+# FIXME: use tarballs rather than git.
+rm -rf $TEST_HOME/.cache
+nix-store --gc # get rid of copies in the store to ensure they get fetched to our git cache
+_NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar
+mv $flake1Dir $flake1Dir.tmp
+mv $flake2Dir $flake2Dir.tmp
+nix-store --gc
+_NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar
+_NIX_FORCE_HTTP=1 nix build -o $TEST_ROOT/result git+file://$flake2Dir#bar --refresh
+mv $flake1Dir.tmp $flake1Dir
+mv $flake2Dir.tmp $flake2Dir
+
+# Add nonFlakeInputs to flake3.
+rm $flake3Dir/flake.nix
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs = {
+ flake1 = {};
+ flake2 = {};
+ nonFlake = {
+ url = git+file://$nonFlakeDir;
+ flake = false;
+ };
+ };
+
+ description = "Fnord";
+
+ outputs = inputs: rec {
+ packages.$system.xyzzy = inputs.flake2.packages.$system.bar;
+ packages.$system.sth = inputs.flake1.packages.$system.foo;
+ packages.$system.fnord =
+ with import ./config.nix;
+ mkDerivation {
+ inherit system;
+ name = "fnord";
+ buildCommand = ''
+ cat \${inputs.nonFlake}/README.md > \$out
+ '';
+ };
+ };
+}
+EOF
+
+cp ./config.nix $flake3Dir
+
+git -C $flake3Dir add flake.nix config.nix
+git -C $flake3Dir commit -m 'Add nonFlakeInputs'
+
+# Check whether `nix build` works with a lockfile which is missing a
+# nonFlakeInputs.
+nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file
+
+nix build -o $TEST_ROOT/result flake3#fnord
+[[ $(cat $TEST_ROOT/result) = FNORD ]]
+
+# Check whether flake input fetching is lazy: flake3#sth does not
+# depend on flake2, so this shouldn't fail.
+rm -rf $TEST_HOME/.cache
+clearStore
+mv $flake2Dir $flake2Dir.tmp
+mv $nonFlakeDir $nonFlakeDir.tmp
+nix build -o $TEST_ROOT/result flake3#sth
+(! nix build -o $TEST_ROOT/result flake3#xyzzy)
+(! nix build -o $TEST_ROOT/result flake3#fnord)
+mv $flake2Dir.tmp $flake2Dir
+mv $nonFlakeDir.tmp $nonFlakeDir
+nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord
+
+# Test doing multiple `lookupFlake`s
+nix build -o $TEST_ROOT/result flake4#xyzzy
+
+# Test 'nix flake update' and --override-flake.
+nix flake update $flake3Dir
+[[ -z $(git -C $flake3Dir diff master) ]]
+
+nix flake update $flake3Dir --recreate-lock-file --override-flake flake2 nixpkgs
+[[ ! -z $(git -C $flake3Dir diff master) ]]
+
+# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore
+git -C $flake3Dir checkout -b removeXyzzy
+rm $flake3Dir/flake.nix
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs = {
+ nonFlake = {
+ url = "$nonFlakeDir";
+ flake = false;
+ };
+ };
+
+ description = "Fnord";
+
+ outputs = { self, flake1, flake2, nonFlake }: rec {
+ packages.$system.sth = flake1.packages.$system.foo;
+ packages.$system.fnord =
+ with import ./config.nix;
+ mkDerivation {
+ inherit system;
+ name = "fnord";
+ buildCommand = ''
+ cat \${nonFlake}/README.md > \$out
+ '';
+ };
+ };
+}
+EOF
+nix flake update $flake3Dir
+git -C $flake3Dir add flake.nix flake.lock
+git -C $flake3Dir commit -m 'Remove packages.xyzzy'
+git -C $flake3Dir checkout master
+
+# Test whether fuzzy-matching works for registry entries.
+(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy)
+nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth
+
+# Testing the nix CLI
+nix registry add flake1 flake3
+[[ $(nix registry list | wc -l) == 8 ]]
+nix registry pin flake1
+[[ $(nix registry list | wc -l) == 8 ]]
+nix registry remove flake1
+[[ $(nix registry list | wc -l) == 7 ]]
+
+# Test 'nix flake init'.
+cat > $templatesDir/flake.nix <<EOF
+{
+ description = "Some templates";
+
+ outputs = { self }: {
+ templates = {
+ trivial = {
+ path = ./trivial;
+ description = "A trivial flake";
+ };
+ };
+ defaultTemplate = self.templates.trivial;
+ };
+}
+EOF
+
+mkdir $templatesDir/trivial
+
+cat > $templatesDir/trivial/flake.nix <<EOF
+{
+ description = "A flake for building Hello World";
+
+ outputs = { self, nixpkgs }: {
+ packages.x86_64-linux.hello = nixpkgs.legacyPackages.x86_64-linux.hello;
+ defaultPackage.x86_64-linux = self.packages.x86_64-linux.hello;
+ };
+}
+EOF
+
+git -C $templatesDir add flake.nix trivial/flake.nix
+git -C $templatesDir commit -m 'Initial'
+
+nix flake check templates
+nix flake show templates
+
+(cd $flake7Dir && nix flake init)
+(cd $flake7Dir && nix flake init) # check idempotence
+git -C $flake7Dir add flake.nix
+nix flake check $flake7Dir
+nix flake show $flake7Dir
+git -C $flake7Dir commit -a -m 'Initial'
+
+# Test 'nix flake new'.
+rm -rf $flake6Dir
+nix flake new -t templates#trivial $flake6Dir
+nix flake new -t templates#trivial $flake6Dir # check idempotence
+nix flake check $flake6Dir
+
+# Test 'nix flake clone'.
+rm -rf $TEST_ROOT/flake1-v2
+nix flake clone flake1 --dest $TEST_ROOT/flake1-v2
+[ -e $TEST_ROOT/flake1-v2/flake.nix ]
+
+# More 'nix flake check' tests.
+cat > $flake3Dir/flake.nix <<EOF
+{
+ outputs = { flake1, self }: {
+ overlay = final: prev: {
+ };
+ };
+}
+EOF
+
+nix flake check $flake3Dir
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ outputs = { flake1, self }: {
+ overlay = finalll: prev: {
+ };
+ };
+}
+EOF
+
+(! nix flake check $flake3Dir)
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ outputs = { flake1, self }: {
+ nixosModules.foo = {
+ a.b.c = 123;
+ foo = true;
+ };
+ };
+}
+EOF
+
+nix flake check $flake3Dir
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ outputs = { flake1, self }: {
+ nixosModules.foo = {
+ a.b.c = 123;
+ foo = assert false; true;
+ };
+ };
+}
+EOF
+
+(! nix flake check $flake3Dir)
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ outputs = { flake1, self }: {
+ nixosModule = { config, pkgs, ... }: {
+ a.b.c = 123;
+ };
+ };
+}
+EOF
+
+nix flake check $flake3Dir
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ outputs = { flake1, self }: {
+ nixosModule = { config, pkgs }: {
+ a.b.c = 123;
+ };
+ };
+}
+EOF
+
+(! nix flake check $flake3Dir)
+
+# Test 'follows' inputs.
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs.foo = {
+ type = "indirect";
+ id = "flake1";
+ };
+ inputs.bar.follows = "foo";
+
+ outputs = { self, foo, bar }: {
+ };
+}
+EOF
+
+nix flake update $flake3Dir
+[[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["foo"]' ]]
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs.bar.follows = "flake2/flake1";
+
+ outputs = { self, flake2, bar }: {
+ };
+}
+EOF
+
+nix flake update $flake3Dir
+[[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2","flake1"]' ]]
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs.bar.follows = "flake2";
+
+ outputs = { self, flake2, bar }: {
+ };
+}
+EOF
+
+nix flake update $flake3Dir
+[[ $(jq -c .nodes.root.inputs.bar $flake3Dir/flake.lock) = '["flake2"]' ]]
+
+# Test overriding inputs of inputs.
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs.flake2.inputs.flake1 = {
+ type = "git";
+ url = file://$flake7Dir;
+ };
+
+ outputs = { self, flake2 }: {
+ };
+}
+EOF
+
+nix flake update $flake3Dir
+[[ $(jq .nodes.flake1.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
+
+cat > $flake3Dir/flake.nix <<EOF
+{
+ inputs.flake2.inputs.flake1.follows = "foo";
+ inputs.foo.url = git+file://$flake7Dir;
+
+ outputs = { self, flake2 }: {
+ };
+}
+EOF
+
+nix flake update $flake3Dir --recreate-lock-file
+[[ $(jq -c .nodes.flake2.inputs.flake1 $flake3Dir/flake.lock) =~ '["foo"]' ]]
+[[ $(jq .nodes.foo.locked.url $flake3Dir/flake.lock) =~ flake7 ]]
+
+# Test Mercurial flakes.
+rm -rf $flake5Dir
+hg init $flake5Dir
+
+cat > $flake5Dir/flake.nix <<EOF
+{
+ outputs = { self, flake1 }: {
+ defaultPackage.$system = flake1.defaultPackage.$system;
+
+ expr = assert builtins.pathExists ./flake.lock; 123;
+ };
+}
+EOF
+
+hg add $flake5Dir/flake.nix
+hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Initial commit'
+
+nix build -o $TEST_ROOT/result hg+file://$flake5Dir
+[[ -e $TEST_ROOT/result/hello ]]
+
+(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision)
+
+nix eval hg+file://$flake5Dir#expr
+
+nix eval hg+file://$flake5Dir#expr
+
+(! nix eval hg+file://$flake5Dir#expr --no-allow-dirty)
+
+(! nix flake info --json hg+file://$flake5Dir | jq -e -r .revision)
+
+hg commit --config ui.username=foobar@example.org $flake5Dir -m 'Add lock file'
+
+nix flake info --json hg+file://$flake5Dir --refresh | jq -e -r .revision
+nix flake info --json hg+file://$flake5Dir
+[[ $(nix flake info --json hg+file://$flake5Dir | jq -e -r .revCount) = 1 ]]
+
+nix build -o $TEST_ROOT/result hg+file://$flake5Dir --no-registries --no-allow-dirty
+
+# Test tarball flakes
+tar cfz $TEST_ROOT/flake.tar.gz -C $TEST_ROOT --exclude .hg flake5
+
+nix build -o $TEST_ROOT/result file://$TEST_ROOT/flake.tar.gz
+
+# Building with a tarball URL containing a SRI hash should also work.
+url=$(nix flake info --json file://$TEST_ROOT/flake.tar.gz | jq -r .url)
+[[ $url =~ sha256- ]]
+
+nix build -o $TEST_ROOT/result $url
+
+# Building with an incorrect SRI hash should fail.
+nix build -o $TEST_ROOT/result "file://$TEST_ROOT/flake.tar.gz?narHash=sha256-qQ2Zz4DNHViCUrp6gTS7EE4+RMqFQtUfWF2UNUtJKS0=" 2>&1 | grep 'NAR hash mismatch'
+
+# Test --override-input.
+git -C $flake3Dir reset --hard
+nix flake update $flake3Dir --override-input flake2/flake1 flake5 -vvvvv
+[[ $(jq .nodes.flake1_2.locked.url $flake3Dir/flake.lock) =~ flake5 ]]
+
+nix flake update $flake3Dir --override-input flake2/flake1 flake1
+[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
+
+nix flake update $flake3Dir --override-input flake2/flake1 flake1/master/$hash1
+[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash1 ]]
+
+# Test --update-input.
+nix flake update $flake3Dir
+[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) = $hash1 ]]
+
+nix flake update $flake3Dir --update-input flake2/flake1
+[[ $(jq -r .nodes.flake1_2.locked.rev $flake3Dir/flake.lock) =~ $hash2 ]]
+
+# Test 'nix flake list-inputs'.
+[[ $(nix flake list-inputs $flake3Dir | wc -l) == 5 ]]
+nix flake list-inputs $flake3Dir --json | jq .
+
+# Test circular flake dependencies.
+cat > $flakeA/flake.nix <<EOF
+{
+ inputs.b.url = git+file://$flakeB;
+ inputs.b.inputs.a.follows = "/";
+
+ outputs = { self, nixpkgs, b }: {
+ foo = 123 + b.bar;
+ xyzzy = 1000;
+ };
+}
+EOF
+
+git -C $flakeA add flake.nix
+
+cat > $flakeB/flake.nix <<EOF
+{
+ inputs.a.url = git+file://$flakeA;
+
+ outputs = { self, nixpkgs, a }: {
+ bar = 456 + a.xyzzy;
+ };
+}
+EOF
+
+git -C $flakeB add flake.nix
+git -C $flakeB commit -a -m 'Foo'
+
+[[ $(nix eval $flakeA#foo) = 1579 ]]
+[[ $(nix eval $flakeA#foo) = 1579 ]]
+
+sed -i $flakeB/flake.nix -e 's/456/789/'
+git -C $flakeB commit -a -m 'Foo'
+
+[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]]
+
+# Test list-inputs with circular dependencies
+nix flake list-inputs $flakeA
diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index de1e2cfe4..3add896c6 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -13,24 +13,32 @@ fake_free=$TEST_ROOT/fake-free
export _NIX_TEST_FREE_SPACE_FILE=$fake_free
echo 1100 > $fake_free
+fifoLock=$TEST_ROOT/fifoLock
+mkfifo "$fifoLock"
+
expr=$(cat <<EOF
with import ./config.nix; mkDerivation {
name = "gc-A";
buildCommand = ''
set -x
[[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]]
+
mkdir \$out
echo foo > \$out/bar
- echo 1...
- sleep 2
- echo 200 > ${fake_free}.tmp1
+
+ # Pretend that we run out of space
+ echo 100 > ${fake_free}.tmp1
mv ${fake_free}.tmp1 $fake_free
- echo 2...
- sleep 2
- echo 3...
- sleep 2
- echo 4...
- [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]
+
+ # Wait for the GC to run
+ for i in {1..20}; do
+ echo ''\${i}...
+ if [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]]; then
+ exit 0
+ fi
+ sleep 1
+ done
+ exit 1
'';
}
EOF
@@ -43,28 +51,29 @@ with import ./config.nix; mkDerivation {
set -x
mkdir \$out
echo foo > \$out/bar
- echo 1...
- sleep 2
- echo 200 > ${fake_free}.tmp2
- mv ${fake_free}.tmp2 $fake_free
- echo 2...
- sleep 2
- echo 3...
- sleep 2
- echo 4...
+
+ # Wait for the first build to finish
+ cat "$fifoLock"
'';
}
EOF
)
-nix build -v -o $TEST_ROOT/result-A -L "($expr)" \
+nix build --impure -v -o $TEST_ROOT/result-A -L --expr "$expr" \
--min-free 1000 --max-free 2000 --min-free-check-interval 1 &
-pid=$!
+pid1=$!
-nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \
- --min-free 1000 --max-free 2000 --min-free-check-interval 1
+nix build --impure -v -o $TEST_ROOT/result-B -L --expr "$expr2" \
+ --min-free 1000 --max-free 2000 --min-free-check-interval 1 &
+pid2=$!
-wait "$pid"
+# Once the first build is done, unblock the second one.
+# If the first build fails, we need to postpone the failure to still allow
+# the second one to finish
+wait "$pid1" || FIRSTBUILDSTATUS=$?
+echo "unlock" > $fifoLock
+( exit ${FIRSTBUILDSTATUS:-0} )
+wait "$pid2"
[[ foo = $(cat $TEST_ROOT/result-A/bar) ]]
[[ foo = $(cat $TEST_ROOT/result-B/bar) ]]
diff --git a/tests/gc-concurrent.builder.sh b/tests/gc-concurrent.builder.sh
index 0cd67df3a..bb6dcd4cf 100644
--- a/tests/gc-concurrent.builder.sh
+++ b/tests/gc-concurrent.builder.sh
@@ -1,7 +1,10 @@
+echo "Build started" > "$lockFifo"
+
mkdir $out
echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar
-sleep 10
+# Wait for someone to write on the fifo
+cat "$lockFifo"
# $out should not have been GC'ed while we were sleeping, but just in
# case...
diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix
index 21671ea2c..0aba1f983 100644
--- a/tests/gc-concurrent.nix
+++ b/tests/gc-concurrent.nix
@@ -1,5 +1,7 @@
with import ./config.nix;
+{ lockFifo ? null }:
+
rec {
input1 = mkDerivation {
@@ -16,6 +18,7 @@ rec {
name = "gc-concurrent";
builder = ./gc-concurrent.builder.sh;
inherit input1 input2;
+ inherit lockFifo;
};
test2 = mkDerivation {
diff --git a/tests/gc-concurrent.sh b/tests/gc-concurrent.sh
index d395930ca..2c6622c62 100644
--- a/tests/gc-concurrent.sh
+++ b/tests/gc-concurrent.sh
@@ -2,7 +2,10 @@ source common.sh
clearStore
-drvPath1=$(nix-instantiate gc-concurrent.nix -A test1)
+lockFifo1=$TEST_ROOT/test1.fifo
+mkfifo "$lockFifo1"
+
+drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1")
outPath1=$(nix-store -q $drvPath1)
drvPath2=$(nix-instantiate gc-concurrent.nix -A test2)
@@ -22,19 +25,16 @@ ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2
nix-store -rvv "$drvPath1" &
pid1=$!
-# Start build #2 in the background after 10 seconds.
-(sleep 10 && nix-store -rvv "$drvPath2") &
-pid2=$!
+# Wait for the build of $drvPath1 to start
+cat $lockFifo1
# Run the garbage collector while the build is running.
-sleep 6
nix-collect-garbage
-# Wait for build #1/#2 to finish.
+# Unlock the build of $drvPath1
+echo "" > $lockFifo1
echo waiting for pid $pid1 to finish...
wait $pid1
-echo waiting for pid $pid2 to finish...
-wait $pid2
# Check that the root of build #1 and its dependencies haven't been
# deleted. The should not be deleted by the GC because they were
@@ -42,8 +42,9 @@ wait $pid2
cat $outPath1/foobar
cat $outPath1/input-2/bar
-# Check that build #2 has succeeded. It should succeed because the
-# derivation is a GC root.
+# Check that the build build $drvPath2 succeeds.
+# It should succeed because the derivation is a GC root.
+nix-store -rvv "$drvPath2"
cat $outPath2/foobar
rm -f "$NIX_STATE_DIR"/gcroots/foo*
diff --git a/tests/gc-concurrent2.builder.sh b/tests/gc-concurrent2.builder.sh
index 4bfb33103..4f6c58b96 100644
--- a/tests/gc-concurrent2.builder.sh
+++ b/tests/gc-concurrent2.builder.sh
@@ -3,5 +3,3 @@ echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar
# Check that the GC hasn't deleted the lock on our output.
test -e "$out.lock"
-
-sleep 6
diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix
new file mode 100644
index 000000000..a47610d9a
--- /dev/null
+++ b/tests/github-flakes.nix
@@ -0,0 +1,148 @@
+{ nixpkgs, system, overlay }:
+
+with import (nixpkgs + "/nixos/lib/testing.nix") {
+ inherit system;
+ extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
+};
+
+let
+
+ # Generate a fake root CA and a fake github.com certificate.
+ cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; }
+ ''
+ mkdir -p $out
+
+ openssl genrsa -out ca.key 2048
+ openssl req -new -x509 -days 36500 -key ca.key \
+ -subj "/C=NL/ST=Denial/L=Springfield/O=Dis/CN=Root CA" -out $out/ca.crt
+
+ openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \
+ -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr
+ openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:raw.githubusercontent.com") \
+ -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt
+ '';
+
+ registry = pkgs.writeTextFile {
+ name = "registry";
+ text = ''
+ {
+ "flakes": [
+ {
+ "from": {
+ "type": "indirect",
+ "id": "nixpkgs"
+ },
+ "to": {
+ "type": "github",
+ "owner": "NixOS",
+ "repo": "nixpkgs"
+ }
+ }
+ ],
+ "version": 2
+ }
+ '';
+ destination = "/flake-registry.json";
+ };
+
+ api = pkgs.runCommand "nixpkgs-flake" {}
+ ''
+ mkdir -p $out/tarball
+
+ dir=NixOS-nixpkgs-${nixpkgs.shortRev}
+ cp -prd ${nixpkgs} $dir
+ # Set the correct timestamp in the tarball.
+ find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} --
+ tar cfz $out/tarball/${nixpkgs.rev} $dir --hard-dereference
+
+ mkdir -p $out/commits
+ echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD
+ '';
+
+in
+
+makeTest (
+
+{
+
+ nodes =
+ { # Impersonate github.com and api.github.com.
+ github =
+ { config, pkgs, ... }:
+ { networking.firewall.allowedTCPPorts = [ 80 443 ];
+
+ services.httpd.enable = true;
+ services.httpd.adminAddr = "foo@example.org";
+ services.httpd.extraConfig = ''
+ ErrorLog syslog:local6
+ '';
+ services.httpd.virtualHosts."github.com" =
+ { forceSSL = true;
+ sslServerKey = "${cert}/server.key";
+ sslServerCert = "${cert}/server.crt";
+ servedDirs =
+ [ { urlPath = "/NixOS/flake-registry/raw/master";
+ dir = registry;
+ }
+ ];
+ };
+ services.httpd.virtualHosts."api.github.com" =
+ { forceSSL = true;
+ sslServerKey = "${cert}/server.key";
+ sslServerCert = "${cert}/server.crt";
+ servedDirs =
+ [ { urlPath = "/repos/NixOS/nixpkgs";
+ dir = api;
+ }
+ ];
+ };
+ };
+
+ client =
+ { config, lib, pkgs, nodes, ... }:
+ { virtualisation.writableStore = true;
+ virtualisation.diskSize = 2048;
+ virtualisation.pathsInNixDB = [ pkgs.hello pkgs.fuse ];
+ virtualisation.memorySize = 4096;
+ nix.binaryCaches = lib.mkForce [ ];
+ nix.extraOptions = "experimental-features = nix-command flakes";
+ environment.systemPackages = [ pkgs.jq ];
+ networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} =
+ [ "github.com" "api.github.com" "raw.githubusercontent.com" ];
+ security.pki.certificateFiles = [ "${cert}/ca.crt" ];
+ };
+ };
+
+ testScript = { nodes }:
+ ''
+ use POSIX qw(strftime);
+
+ startAll;
+
+ $github->waitForUnit("httpd.service");
+
+ $client->succeed("curl -v https://github.com/ >&2");
+
+ $client->succeed("nix registry list | grep nixpkgs");
+
+ $client->succeed("nix flake info nixpkgs --json | jq -r .revision") eq "${nixpkgs.rev}\n"
+ or die "revision mismatch";
+
+ $client->succeed("nix registry pin nixpkgs");
+
+ $client->succeed("nix flake info nixpkgs --tarball-ttl 0 >&2");
+
+ # Shut down the web server. The flake should be cached on the client.
+ $github->succeed("systemctl stop httpd.service");
+
+ my $date = $client->succeed("nix flake info nixpkgs --json | jq -M .lastModified");
+ strftime("%Y%m%d%H%M%S", gmtime($date)) eq "${nixpkgs.lastModifiedDate}" or die "time mismatch";
+
+ $client->succeed("nix build nixpkgs#hello");
+
+ # The build shouldn't fail even with --tarball-ttl 0 (the server
+ # being down should not be a fatal error).
+ $client->succeed("nix build nixpkgs#fuse --tarball-ttl 0");
+ '';
+
+})
diff --git a/tests/init.sh b/tests/init.sh
index c62c4856a..f9ced6b0d 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -18,6 +18,8 @@ build-users-group =
keep-derivations = false
sandbox = false
experimental-features = nix-command flakes
+gc-reserved-space = 0
+flake-registry = $TEST_ROOT/registry.json
include nix.conf.extra
EOF
diff --git a/tests/local-store.sh b/tests/local-store.sh
new file mode 100644
index 000000000..4ec3d64b0
--- /dev/null
+++ b/tests/local-store.sh
@@ -0,0 +1,20 @@
+source common.sh
+
+cd $TEST_ROOT
+
+echo example > example.txt
+mkdir -p ./x
+
+NIX_STORE_DIR=$TEST_ROOT/x
+
+CORRECT_PATH=$(nix-store --store ./x --add example.txt)
+
+PATH1=$(nix path-info --store ./x $CORRECT_PATH)
+[ $CORRECT_PATH == $PATH1 ]
+
+PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH)
+[ $CORRECT_PATH == $PATH2 ]
+
+# FIXME we could also test the query parameter version:
+# PATH3=$(nix path-info --store "local?store=$PWD/x" $CORRECT_PATH)
+# [ $CORRECT_PATH == $PATH3 ]
diff --git a/tests/local.mk b/tests/local.mk
index 536661af8..0f3bfe606 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -6,7 +6,7 @@ nix_tests = \
gc-auto.sh \
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
gc-runtime.sh check-refs.sh filter-source.sh \
- remote-store.sh export.sh export-graph.sh \
+ local-store.sh remote-store.sh export.sh export-graph.sh \
timeout.sh secure-drv-outputs.sh nix-channel.sh \
multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \
binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \
@@ -31,13 +31,14 @@ nix_tests = \
nix-copy-ssh.sh \
post-hook.sh \
function-trace.sh \
- recursive.sh
+ recursive.sh \
+ flakes.sh
# parallel.sh
install-tests += $(foreach x, $(nix_tests), tests/$(x))
tests-environment = NIX_REMOTE= $(bash) -e
-clean-files += $(d)/common.sh
+clean-files += $(d)/common.sh $(d)/config.nix
-installcheck: $(d)/common.sh $(d)/config.nix $(d)/plugins/libplugintest.$(SO_EXT)
+test-deps += tests/common.sh tests/config.nix tests/plugins/libplugintest.$(SO_EXT)
diff --git a/tests/misc.sh b/tests/misc.sh
index fd4908e25..a81c9dbb1 100644
--- a/tests/misc.sh
+++ b/tests/misc.sh
@@ -16,6 +16,11 @@ nix-env --foo 2>&1 | grep "no operation"
nix-env -q --foo 2>&1 | grep "unknown flag"
# Eval Errors.
-eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
-echo $eval_res | grep "(string) (1:15)"
-echo $eval_res | grep "infinite recursion encountered"
+eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
+echo $eval_arg_res | grep "at: (1:15) from string"
+echo $eval_arg_res | grep "infinite recursion encountered"
+
+eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true)
+echo $eval_stdin_res | grep "at: (1:15) from stdin"
+echo $eval_stdin_res | grep "infinite recursion encountered"
+
diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index bb5db7410..9c9d119b7 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -1,8 +1,11 @@
# Test ‘nix-copy-closure’.
-{ nixpkgs, system, nix }:
+{ nixpkgs, system, overlay }:
-with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") {
+ inherit system;
+ extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
+};
makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
@@ -11,7 +14,6 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
{ config, lib, pkgs, ... }:
{ virtualisation.writableStore = true;
virtualisation.pathsInNixDB = [ pkgA ];
- nix.package = nix;
nix.binaryCaches = lib.mkForce [ ];
};
@@ -20,7 +22,6 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
{ services.openssh.enable = true;
virtualisation.writableStore = true;
virtualisation.pathsInNixDB = [ pkgB pkgC ];
- nix.package = nix;
};
};
diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh
index 235e2a5ff..650904057 100644
--- a/tests/nix-shell.sh
+++ b/tests/nix-shell.sh
@@ -55,3 +55,10 @@ chmod a+rx $TEST_ROOT/shell.shebang.rb
output=$($TEST_ROOT/shell.shebang.rb abc ruby)
[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ]
+
+# Test 'nix develop'.
+nix develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]'
+
+# Test 'nix print-dev-env'.
+source <(nix print-dev-env -f shell.nix shellDrv)
+[[ -n $stdenv ]]
diff --git a/tests/path.nix b/tests/path.nix
new file mode 100644
index 000000000..883c3c41b
--- /dev/null
+++ b/tests/path.nix
@@ -0,0 +1,14 @@
+with import ./config.nix;
+
+mkDerivation {
+ name = "filter";
+ builder = builtins.toFile "builder" "ln -s $input $out";
+ input =
+ builtins.path {
+ path = ((builtins.getEnv "TEST_ROOT") + "/filterin");
+ filter = path: type:
+ type != "symlink"
+ && baseNameOf path != "foo"
+ && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path));
+ };
+}
diff --git a/tests/plugins.sh b/tests/plugins.sh
index 4b1baeddc..50bfaf7e9 100644
--- a/tests/plugins.sh
+++ b/tests/plugins.sh
@@ -2,6 +2,6 @@ source common.sh
set -o pipefail
-res=$(nix eval '(builtins.anotherNull)' --option setting-set true --option plugin-files $PWD/plugins/libplugintest*)
+res=$(nix eval --expr builtins.anotherNull --option setting-set true --option plugin-files $PWD/plugins/libplugintest*)
[ "$res"x = "nullx" ]
diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh
index 49c856448..43a765997 100644
--- a/tests/pure-eval.sh
+++ b/tests/pure-eval.sh
@@ -2,17 +2,17 @@ source common.sh
clearStore
-nix eval --pure-eval '(assert 1 + 2 == 3; true)'
+nix eval --expr 'assert 1 + 2 == 3; true'
-[[ $(nix eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]]
+[[ $(nix eval --impure --expr 'builtins.readFile ./pure-eval.sh') =~ clearStore ]]
-(! nix eval --pure-eval '(builtins.readFile ./pure-eval.sh)')
+(! nix eval --expr 'builtins.readFile ./pure-eval.sh')
-(! nix eval --pure-eval '(builtins.currentTime)')
-(! nix eval --pure-eval '(builtins.currentSystem)')
+(! nix eval --expr builtins.currentTime)
+(! nix eval --expr builtins.currentSystem)
(! nix-instantiate --pure-eval ./simple.nix)
-[[ $(nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]]
-(! nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)")
-nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)"
+[[ $(nix eval --impure --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") == 123 ]]
+(! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x")
+nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x"
diff --git a/tests/recursive.sh b/tests/recursive.sh
index 2d4f83895..cf10d55bf 100644
--- a/tests/recursive.sh
+++ b/tests/recursive.sh
@@ -9,7 +9,7 @@ rm -f $TEST_ROOT/result
export unreachable=$(nix add-to-store ./recursive.sh)
-nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L '(
+nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --expr '
with import ./config.nix;
with import <nix/config.nix>;
mkDerivation {
@@ -49,7 +49,7 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu
[[ $(nix $opts path-info --all | wc -l) -eq 3 ]]
# Build a derivation.
- nix $opts build -L '\''(
+ nix $opts build -L --impure --expr '\''
derivation {
name = "inner1";
builder = builtins.getEnv "SHELL";
@@ -57,13 +57,13 @@ nix --experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/resu
fnord = builtins.toFile "fnord" "fnord";
args = [ "-c" "echo $fnord blaat > $out" ];
}
- )'\''
+ '\''
[[ $(nix $opts path-info --json ./result) =~ fnord ]]
ln -s $(nix $opts path-info ./result) $out/inner1
'\'\'';
- })
+ }
'
[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]]
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 18d490830..153956619 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -1,8 +1,11 @@
# Test Nix's remote build feature.
-{ nixpkgs, system, nix }:
+{ nixpkgs, system, overlay }:
-with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") {
+ inherit system;
+ extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
+};
makeTest (
@@ -13,7 +16,6 @@ let
{ config, pkgs, ... }:
{ services.openssh.enable = true;
virtualisation.writableStore = true;
- nix.package = nix;
nix.useSandbox = true;
};
@@ -59,7 +61,6 @@ in
];
virtualisation.writableStore = true;
virtualisation.pathsInNixDB = [ config.system.build.extraUtils ];
- nix.package = nix;
nix.binaryCaches = lib.mkForce [ ];
programs.ssh.extraConfig = "ConnectTimeout 30";
};
diff --git a/tests/restricted.sh b/tests/restricted.sh
index e02becc60..242b901dd 100644
--- a/tests/restricted.sh
+++ b/tests/restricted.sh
@@ -17,18 +17,18 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel'
(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>')
nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=.
-p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)")
+p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)")
cmp $p restricted.sh
-(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval)
+(! nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval)
-(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/")
+(! nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/")
-nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh"
+nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh"
-(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval)
-(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval)
-(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval)
+(! nix eval --raw --expr "builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval)
+(! nix eval --raw --expr "builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval)
+(! nix eval --raw --expr "fetchGit git://github.com/NixOS/patchelf.git" --impure --restrict-eval)
ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix
[[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]]
@@ -37,7 +37,7 @@ ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix
(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .)
nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I .
-[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]]
+[[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]]
# Check whether we can leak symlink information through directory traversal.
traverseDir="$(pwd)/restricted-traverse-me"
@@ -45,7 +45,7 @@ ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent"
mkdir -p "$traverseDir"
goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')"
output="$(nix eval --raw --restrict-eval -I "$traverseDir" \
- "(builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\")" \
+ --expr "builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\"" \
2>&1 || :)"
echo "$output" | grep "is forbidden"
! echo "$output" | grep -F restricted-secret
diff --git a/tests/search.sh b/tests/search.sh
index 14da3127b..ee3261687 100644
--- a/tests/search.sh
+++ b/tests/search.sh
@@ -3,41 +3,23 @@ source common.sh
clearStore
clearCache
-# No packages
-(( $(NIX_PATH= nix search -u|wc -l) == 0 ))
-
-# Haven't updated cache, still nothing
-(( $(nix search -f search.nix hello|wc -l) == 0 ))
-(( $(nix search -f search.nix |wc -l) == 0 ))
-
-# Update cache, search should work
-(( $(nix search -f search.nix -u hello|wc -l) > 0 ))
-
-# Use cache
-(( $(nix search -f search.nix foo|wc -l) > 0 ))
-(( $(nix search foo|wc -l) > 0 ))
-
-# Test --no-cache works
-# No results from cache
-(( $(nix search --no-cache foo |wc -l) == 0 ))
-# Does find results from file pointed at
-(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 ))
+(( $(nix search -f search.nix '' hello | wc -l) > 0 ))
# Check descriptions are searched
-(( $(nix search broken | wc -l) > 0 ))
+(( $(nix search -f search.nix '' broken | wc -l) > 0 ))
# Check search that matches nothing
-(( $(nix search nosuchpackageexists | wc -l) == 0 ))
+(( $(nix search -f search.nix '' nosuchpackageexists | wc -l) == 0 ))
# Search for multiple arguments
-(( $(nix search hello empty | wc -l) == 3 ))
+(( $(nix search -f search.nix '' hello empty | wc -l) == 2 ))
# Multiple arguments will not exist
-(( $(nix search hello broken | wc -l) == 0 ))
+(( $(nix search -f search.nix '' hello broken | wc -l) == 0 ))
## Search expressions
# Check that empty search string matches all
-nix search|grep -q foo
-nix search|grep -q bar
-nix search|grep -q hello
+nix search -f search.nix '' |grep -q foo
+nix search -f search.nix '' |grep -q bar
+nix search -f search.nix '' |grep -q hello
diff --git a/tests/setuid.nix b/tests/setuid.nix
index 63d3c05cb..6f2f7d392 100644
--- a/tests/setuid.nix
+++ b/tests/setuid.nix
@@ -1,15 +1,17 @@
# Verify that Linux builds cannot create setuid or setgid binaries.
-{ nixpkgs, system, nix }:
+{ nixpkgs, system, overlay }:
-with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; };
+with import (nixpkgs + "/nixos/lib/testing.nix") {
+ inherit system;
+ extraConfigurations = [ { nixpkgs.overlays = [ overlay ]; } ];
+};
makeTest {
machine =
{ config, lib, pkgs, ... }:
{ virtualisation.writableStore = true;
- nix.package = nix;
nix.binaryCaches = lib.mkForce [ ];
nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ];
virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ];
diff --git a/tests/shell.nix b/tests/shell.nix
index 6cb4f082b..6ce59b416 100644
--- a/tests/shell.nix
+++ b/tests/shell.nix
@@ -24,6 +24,7 @@ let pkgs = rec {
VAR_FROM_NIX = "bar";
TEST_inNixShell = if inNixShell then "true" else "false";
inherit stdenv;
+ outputs = ["dev" "out"];
};
# Used by nix-shell -p
diff --git a/tests/tarball.sh b/tests/tarball.sh
index b3ec16d40..88a1a07a0 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -27,10 +27,13 @@ test_tarball() {
nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
- nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+ nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
+ nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2
+ nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true'
nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext