diff options
150 files changed, 3957 insertions, 5108 deletions
diff --git a/.gitignore b/.gitignore index de8e9354f..a175e8dfe 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-pull /scripts/nix-push /scripts/nix-switch /scripts/nix-collect-garbage @@ -43,11 +42,8 @@ Makefile.config /scripts/nix-channel /scripts/nix-build /scripts/nix-copy-closure -/scripts/nix-generate-patches /scripts/NixConfig.pm /scripts/NixManifest.pm -/scripts/GeneratePatches.pm -/scripts/download-using-manifests.pl /scripts/copy-from-other-stores.pl /scripts/download-from-binary-cache.pl /scripts/find-runtime-roots.pl @@ -55,10 +51,6 @@ Makefile.config /scripts/nix-reduce-build /scripts/nix-http-export.cgi -# /src/bsdiff-4.3/ -/src/bsdiff-4.3/bsdiff -/src/bsdiff-4.3/bspatch - # /src/libexpr/ /src/libexpr/lexer-tab.cc /src/libexpr/lexer-tab.hh @@ -76,9 +68,6 @@ Makefile.config # /src/nix-instantiate/ /src/nix-instantiate/nix-instantiate -# /src/nix-log2xml/ -/src/nix-log2xml/nix-log2xml - # /src/nix-store/ /src/nix-store/nix-store @@ -12,9 +12,7 @@ makefiles = \ src/nix-daemon/local.mk \ src/nix-collect-garbage/local.mk \ src/download-via-ssh/local.mk \ - src/nix-log2xml/local.mk \ src/nix-prefetch-url/local.mk \ - src/bsdiff-4.3/local.mk \ perl/local.mk \ scripts/local.mk \ corepkgs/local.mk \ diff --git a/configure.ac b/configure.ac index 2a043ed13..1215782a1 100644 --- a/configure.ac +++ b/configure.ac @@ -16,6 +16,8 @@ AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], machine_name="i686";; amd64) machine_name="x86_64";; + armv6|armv7) + machine_name="${host_cpu}l";; *) machine_name="$host_cpu";; esac @@ -11,7 +11,7 @@ exec $s release.nix -A tarball --command " export NIX_PATH='$NIX_PATH' export NIX_BUILD_SHELL=$(type -p bash) export c=\$configureFlags - exec $s release.nix -A build.$(if [ $(uname -s) = Darwin ]; then echo x86_64-darwin; else echo x86_64-linux; fi) --exclude tarball --command ' + exec $s release.nix -A build.$(if [ $(uname -s) = Darwin ]; then echo x86_64-darwin; elif [[ $(uname -m) =~ ^i[3456]86$ ]]; then echo i686-linux; else echo x86_64-linux; fi) --exclude tarball --command ' configureFlags+=\" \$c --prefix=$(pwd)/inst --sysconfdir=$(pwd)/inst/etc\" return '" \ diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index acddd63e1..4c8f3d9d3 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -306,21 +306,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>build-cache-failure</literal></term> - - <listitem><para>If set to <literal>true</literal>, Nix will - “cache” build failures, meaning that it will remember (in its - database) that a derivation previously failed. If you then try to - build the derivation again, Nix will immediately fail rather than - perform the build again. Failures in fixed-output derivations - (such as <function>fetchurl</function> calls) are never cached. - The “failed” status of a derivation can be cleared using - <command>nix-store --clear-failed-paths</command>. By default, - failure caching is disabled.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>build-keep-log</literal></term> <listitem><para>If set to <literal>true</literal> (the default), @@ -421,16 +406,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>binary-cache-secret-key-file</literal></term> - - <listitem><para>Path of the file containing the secret key to be - used for signing binary caches. This file can be generated using - <command>nix-store - --generate-binary-cache-key</command>.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>binary-caches-parallel-connections</literal></term> <listitem><para>The maximum number of parallel HTTP connections @@ -450,18 +425,6 @@ flag, e.g. <literal>--option gc-keep-outputs false</literal>.</para> </varlistentry> - <varlistentry><term><literal>force-manifest</literal></term> - - <listitem><para>If this option is set to <literal>false</literal> - (default) and a Nix channel provides both a manifest and a binary - cache, only the binary cache will be used. If set to - <literal>true</literal>, the manifest will be fetched as well. - This is useful if you want to use binary patches (which are - currently not supported by binary caches).</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>system</literal></term> <listitem><para>This option specifies the canonical Nix system diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index a6f4a2720..0a1f2a8b7 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -73,11 +73,10 @@ condition="manual">See also <xref linkend="sec-channels" <listitem><para>Downloads the Nix expressions of all subscribed channels (or only those included in - <replaceable>names</replaceable> if specified), makes them the + <replaceable>names</replaceable> if specified) and makes them the default for <command>nix-env</command> operations (by symlinking - them from the directory <filename>~/.nix-defexpr</filename>), and - performs a <command>nix-pull</command> on the manifests of all - channels to make pre-built binaries available.</para></listitem> + them from the directory + <filename>~/.nix-defexpr</filename>).</para></listitem> </varlistentry> @@ -187,16 +186,6 @@ following files:</para> </varlistentry> - <varlistentry><term><filename>MANIFEST.bz2</filename></term> - - <listitem><para>(Deprecated in favour of binary caches.) A - manifest as created by <command>nix-push</command>. Only used if - <filename>binary-cache-url</filename> is not present or if the - <filename>nix.conf</filename> option - <option>force-manifest</option> is set.</para></listitem> - - </varlistentry> - </variablelist> </refsection> diff --git a/doc/manual/command-ref/nix-copy-closure.xml b/doc/manual/command-ref/nix-copy-closure.xml index 5848b84a0..97e261ae9 100644 --- a/doc/manual/command-ref/nix-copy-closure.xml +++ b/doc/manual/command-ref/nix-copy-closure.xml @@ -22,7 +22,6 @@ <arg choice='plain'><option>--to</option></arg> <arg choice='plain'><option>--from</option></arg> </group> - <arg><option>--sign</option></arg> <arg><option>--gzip</option></arg> <!-- <arg><option>- -show-progress</option></arg> @@ -87,23 +86,6 @@ those paths. If this bothers you, use </varlistentry> - <varlistentry><term><option>--sign</option></term> - - <listitem><para>Let the sending machine cryptographically sign the - dump of each path with the key in - <filename><replaceable>sysconfdir</replaceable>/nix/signing-key.sec</filename>. - If the user on the target machine does not have direct access to - the Nix store (i.e., if the target machine has a multi-user Nix - installation), then the target machine will check the dump against - <filename><replaceable>sysconfdir</replaceable>/nix/signing-key.pub</filename> - before unpacking it in its Nix store. This allows secure sharing - of store paths between untrusted users on two machines, provided - that there is a trust relation between the Nix installations on - both machines (namely, they have matching public/secret - keys).</para></listitem> - - </varlistentry> - <varlistentry><term><option>--gzip</option></term> <listitem><para>Enable compression of the SSH diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index e9a5f0e09..2ed4a5d9f 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -367,6 +367,10 @@ number of possible ways: linkend="rsec-nix-store-realise">realised</link> and installed.</para></listitem> + <listitem><para>By default all outputs are installed for each derivation. + That can be reduced by setting <literal>meta.outputsToInstall</literal>. + </para></listitem> <!-- TODO: link nixpkgs docs on the ability to override those. --> + </itemizedlist> </para> diff --git a/doc/manual/command-ref/nix-generate-patches.xml b/doc/manual/command-ref/nix-generate-patches.xml deleted file mode 100644 index 70bec432d..000000000 --- a/doc/manual/command-ref/nix-generate-patches.xml +++ /dev/null @@ -1,44 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-generate-patches"> - -<refmeta> - <refentrytitle>nix-generate-patches</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-generate-patches</refname> - <refpurpose>generates binary patches between NAR files</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-generate-patches</command> - <arg choice='plain'><replaceable>NAR-DIR</replaceable></arg> - <arg choice='plain'><replaceable>PATCH-DIR</replaceable></arg> - <arg choice='plain'><replaceable>PATCH-URI</replaceable></arg> - <arg choice='plain'><replaceable>OLD-MANIFEST</replaceable></arg> - <arg choice='plain'><replaceable>NEW-MANIFEST</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<para>The command <command>nix-generate-patches</command> generates -binary patches between NAR files listed in OLD-MANIFEST and NEW-MANIFEST. -The patches are written to the directory PATCH-DIR, and the prefix -PATCH-URI is used to generate URIs for the patches. The patches are -added to NEW-MANIFEST. All NARs are required to exist in NAR-DIR. -Patches are generated between succeeding versions of packages with -the same name.</para> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-install-package.xml b/doc/manual/command-ref/nix-install-package.xml index f7802a95d..e17166caa 100644 --- a/doc/manual/command-ref/nix-install-package.xml +++ b/doc/manual/command-ref/nix-install-package.xml @@ -146,9 +146,7 @@ The elements are as follows: <varlistentry><term><replaceable>manifestURL</replaceable></term> - <listitem><para>The manifest to be pulled by - <command>nix-pull</command>. The manifest must contain - <replaceable>outPath</replaceable>.</para></listitem> + <listitem><para>Obsolete.</para></listitem> </varlistentry> diff --git a/doc/manual/command-ref/nix-pull.xml b/doc/manual/command-ref/nix-pull.xml deleted file mode 100644 index eb471677b..000000000 --- a/doc/manual/command-ref/nix-pull.xml +++ /dev/null @@ -1,54 +0,0 @@ -<refentry xmlns="http://docbook.org/ns/docbook" - xmlns:xlink="http://www.w3.org/1999/xlink" - xmlns:xi="http://www.w3.org/2001/XInclude" - version="5.0" - xml:id="sec-nix-pull"> - -<refmeta> - <refentrytitle>nix-pull</refentrytitle> - <manvolnum>1</manvolnum> - <refmiscinfo class="source">Nix</refmiscinfo> - <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> -</refmeta> - -<refnamediv> - <refname>nix-pull</refname> - <refpurpose>register availability of pre-built binaries (deprecated)</refpurpose> -</refnamediv> - -<refsynopsisdiv> - <cmdsynopsis> - <command>nix-pull</command> - <arg choice='plain'><replaceable>url</replaceable></arg> - </cmdsynopsis> -</refsynopsisdiv> - - -<refsection><title>Description</title> - -<note><para>This command and the use of manifests is deprecated. It is -better to use binary caches.</para></note> - -<para>The command <command>nix-pull</command> obtains a list of -pre-built store paths from the URL <replaceable>url</replaceable>, and -for each of these store paths, registers a substitute derivation that -downloads and unpacks it into the Nix store. This is used to speed up -installations: if you attempt to install something that has already -been built and stored into the network cache, Nix can transparently -re-use the pre-built store paths.</para> - -<para>The file at <replaceable>url</replaceable> must be compatible -with the files created by <replaceable>nix-push</replaceable>.</para> - -</refsection> - - -<refsection><title>Examples</title> - -<screen> -$ nix-pull https://nixos.org/releases/nixpkgs/nixpkgs-15.05pre54468.69858d7/MANIFEST</screen> - -</refsection> - - -</refentry> diff --git a/doc/manual/command-ref/nix-push.xml b/doc/manual/command-ref/nix-push.xml index b8156b455..0749824a0 100644 --- a/doc/manual/command-ref/nix-push.xml +++ b/doc/manual/command-ref/nix-push.xml @@ -73,8 +73,7 @@ automatically.</para> <listitem><para>Optionally, a single <emphasis>manifest</emphasis> file is created that contains the same metadata as the <filename>.narinfo</filename> files. This is for compatibility with - Nix versions prior to 1.2 (see <command>nix-pull</command> for - details).</para></listitem> + Nix versions prior to 1.2.</para></listitem> <listitem><para>A file named <option>nix-cache-info</option> is placed in the destination directory. The existence of this file @@ -135,7 +134,7 @@ automatically.</para> <varlistentry><term><option>--manifest</option></term> <listitem><para>Force the generation of a manifest suitable for - use by <command>nix-pull</command>. The manifest is stored as + use by old versions of Nix. The manifest is stored as <filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem> </varlistentry> @@ -203,20 +202,6 @@ $ nix-push --dest /tmp/cache $(nix-instantiate -A thunderbird) </para> -<para>To generate a manifest suitable for <command>nix-pull</command>: - -<screen> -$ nix-push --dest /tmp/cache $(nix-build -A thunderbird) --manifest -</screen> - -On another machine you can then do: - -<screen> -$ nix-pull http://example.org/cache -</screen> - -to cause the binaries to be used by subsequent Nix operations.</para> - <para>To generate a signed binary cache, you must first generate a key pair, in this example called <literal>cache.example.org-1</literal>, storing the secret key in <filename>./sk</filename> and the public key diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index 58a331179..340f61210 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -1350,82 +1350,6 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil <!--######################################################################--> -<refsection><title>Operation <option>--query-failed-paths</option></title> - -<refsection> - <title>Synopsis</title> - <cmdsynopsis> - <command>nix-store</command> - <arg choice='plain'><option>--query-failed-paths</option></arg> - </cmdsynopsis> -</refsection> - -<refsection><title>Description</title> - -<para>If build failure caching is enabled through the -<literal>build-cache-failure</literal> configuration option, the -operation <option>--query-failed-paths</option> will print out all -store paths that have failed to build.</para> - -</refsection> - -<refsection><title>Example</title> - -<screen> -$ nix-store --query-failed-paths -/nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59 -/nix/store/0011iy7sfwbc1qj5a1f6ifjnbcdail8a-haskell-gitit-ghc7.0.4-0.8.1 -/nix/store/001c0yn1hkh86gprvrb46cxnz3pki7q3-gamin-0.1.10 -<replaceable>…</replaceable> -</screen> - -</refsection> - -</refsection> - - -<!--######################################################################--> - -<refsection><title>Operation <option>--clear-failed-paths</option></title> - -<refsection> - <title>Synopsis</title> - <cmdsynopsis> - <command>nix-store</command> - <arg choice='plain'><option>--clear-failed-paths</option></arg> - <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> - </cmdsynopsis> -</refsection> - -<refsection><title>Description</title> - -<para>If build failure caching is enabled through the -<literal>build-cache-failure</literal> configuration option, the -operation <option>--clear-failed-paths</option> clears the “failed” -state of the given store paths, allowing them to be built again. This -is useful if the failure was actually transient (e.g. because the disk -was full).</para> - -<para>If a path denotes a derivation, its output paths are cleared. -You can provide the argument <literal>*</literal> to clear all store -paths.</para> - -</refsection> - -<refsection><title>Example</title> - -<screen> -$ nix-store --clear-failed-paths /nix/store/000zi5dcla86l92jn1g997jb06sidm7x-perl-PerlMagick-6.59 -$ nix-store --clear-failed-paths * -</screen> - -</refsection> - -</refsection> - - -<!--######################################################################--> - <refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title> <refsection> diff --git a/doc/manual/command-ref/opt-common-syn.xml b/doc/manual/command-ref/opt-common-syn.xml index d65f4009e..5b7936393 100644 --- a/doc/manual/command-ref/opt-common-syn.xml +++ b/doc/manual/command-ref/opt-common-syn.xml @@ -31,7 +31,6 @@ <arg><option>-K</option></arg> <arg><option>--fallback</option></arg> <arg><option>--readonly-mode</option></arg> -<arg><option>--log-type</option> <replaceable>type</replaceable></arg> <arg><option>--show-trace</option></arg> <arg> <option>-I</option> diff --git a/doc/manual/command-ref/opt-common.xml b/doc/manual/command-ref/opt-common.xml index c7e8ae1ed..bc26a9061 100644 --- a/doc/manual/command-ref/opt-common.xml +++ b/doc/manual/command-ref/opt-common.xml @@ -201,61 +201,6 @@ </varlistentry> -<varlistentry xml:id="opt-log-type"><term><option>--log-type</option> -<replaceable>type</replaceable></term> - - <listitem> - - <para>This option determines how the output written to standard - error is formatted. Nix’s diagnostic messages are typically - <emphasis>nested</emphasis>. For instance, when tracing Nix - expression evaluation (<command>nix-env -vvvvv</command>, messages - from subexpressions are nested inside their parent expressions. Nix - builder output is also often nested. For instance, the Nix Packages - generic builder nests the various build tasks (unpack, configure, - compile, etc.), and the GNU Make in <literal>stdenv-linux</literal> - has been patched to provide nesting for recursive Make - invocations.</para> - - <para><replaceable>type</replaceable> can be one of the - following: - - <variablelist> - - <varlistentry><term><literal>pretty</literal></term> - - <listitem><para>Pretty-print the output, indicating different - nesting levels using spaces. This is the - default.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>escapes</literal></term> - - <listitem><para>Indicate nesting using escape codes that can be - interpreted by the <command>nix-log2xml</command> tool in the - Nix source distribution. The resulting XML file can be fed into - the <command>log2html.xsl</command> stylesheet to create an HTML - file that can be browsed interactively, using JavaScript to - expand and collapse parts of the output.</para></listitem> - - </varlistentry> - - <varlistentry><term><literal>flat</literal></term> - - <listitem><para>Remove all nesting.</para></listitem> - - </varlistentry> - - </variablelist> - - </para> - - </listitem> - -</varlistentry> - - <varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term> <listitem><para>This option is accepted by diff --git a/doc/manual/command-ref/utilities.xml b/doc/manual/command-ref/utilities.xml index be2fe6e2d..25e457e4e 100644 --- a/doc/manual/command-ref/utilities.xml +++ b/doc/manual/command-ref/utilities.xml @@ -13,14 +13,10 @@ work with Nix.</para> <xi:include href="nix-collect-garbage.xml" /> <xi:include href="nix-copy-closure.xml" /> <xi:include href="nix-daemon.xml" /> -<!-- -<xi:include href="nix-generate-patches.xml" /> ---> <xi:include href="nix-hash.xml" /> <xi:include href="nix-install-package.xml" /> <xi:include href="nix-instantiate.xml" /> <xi:include href="nix-prefetch-url.xml" /> -<xi:include href="nix-pull.xml" /> <xi:include href="nix-push.xml" /> </chapter> diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 3d7e7fed9..52f2884ab 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -39,7 +39,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid # Generate man pages. man-pages := $(foreach n, \ nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ - nix-collect-garbage.1 nix-push.1 nix-pull.1 \ + nix-collect-garbage.1 nix-push.1 \ nix-prefetch-url.1 nix-channel.1 \ nix-install-package.1 nix-hash.1 nix-copy-closure.1 \ nix.conf.5 nix-daemon.8, \ @@ -76,17 +76,3 @@ all: $(d)/manual.html clean-files += $(d)/manual.html dist-files += $(d)/manual.html - - -# Generate the PDF manual. -$(d)/manual.pdf: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid - $(trace-gen) if test "$(dblatex)" != ""; then \ - cd doc/manual && $(XSLTPROC) --xinclude --stringparam profile.condition manual \ - $(docbookxsl)/profiling/profile.xsl manual.xml | \ - $(dblatex) -o $(notdir $@) $(dblatex_opts) -; \ - else \ - echo "Please install dblatex and rerun configure."; \ - exit 1; \ - fi - -clean-files += $(d)/manual.pdf diff --git a/doc/signing.txt b/doc/signing.txt deleted file mode 100644 index 7403cac47..000000000 --- a/doc/signing.txt +++ /dev/null @@ -1,24 +0,0 @@ -Generate a private key: - -$ (umask 277 && openssl genrsa -out /etc/nix/signing-key.sec 2048) - -The private key should be kept secret (only readable to the Nix daemon -user). - - -Generate the corresponding public key: - -$ openssl rsa -in /etc/nix/signing-key.sec -pubout > /etc/nix/signing-key.pub - -The public key should be copied to all machines to which you want to -export store paths. - - -Signing: - -$ nix-hash --type sha256 --flat svn.nar | openssl rsautl -sign -inkey mykey.sec > svn.nar.sign - - -Verifying a signature: - -$ test "$(nix-hash --type sha256 --flat svn.nar)" = "$(openssl rsautl -verify -inkey mykey.pub -pubin -in svn.nar.sign)" diff --git a/misc/docker/Dockerfile b/misc/docker/Dockerfile index 342c28eda..bfa0c34d0 100644 --- a/misc/docker/Dockerfile +++ b/misc/docker/Dockerfile @@ -1,23 +1,20 @@ -FROM busybox +FROM alpine -RUN set -x \ - && wget -O- http://nixos.org/releases/nix/nix-1.9/nix-1.9-x86_64-linux.tar.bz2 | \ - bzcat - | tar xf - \ - && echo "nixbld:x:30000:nixbld1,nixbld10,nixbld2,nixbld3,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9" >> /etc/group \ - && for i in $(seq 1 9); do echo "nixbld$i:x:3000$i:30000:::" >> /etc/passwd; done \ - && sed -i 's/\$HOME\/\.nix-profile\/etc\/ssl\/certs\/ca-bundle\.crt/\$HOME\/\.nix-profile\/etc\/ca-bundle\.crt/g' nix-1.9-x86_64-linux/install \ - && mkdir -m 0755 /nix && USER=root sh nix-1.9-x86_64-linux/install \ +RUN wget -O- http://nixos.org/releases/nix/nix-1.11.2/nix-1.11.2-x86_64-linux.tar.bz2 | bzcat - | tar xf - \ + && echo "nixbld:x:30000:nixbld1,nixbld2,nixbld3,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld30" >> /etc/group \ + && for i in $(seq 1 30); do echo "nixbld$i:x:$((30000 + $i)):30000:::" >> /etc/passwd; done \ + && mkdir -m 0755 /nix && USER=root sh nix-*-x86_64-linux/install \ && echo ". /root/.nix-profile/etc/profile.d/nix.sh" >> /etc/profile \ - && rm -r /nix-1.9-x86_64-linux + && rm -r /nix-*-x86_64-linux ONBUILD ENV \ ENV=/etc/profile \ PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/root/.nix-profile/etc/ca-bundle.crt \ - SSL_CERT_FILE=/root/.nix-profile/etc/ca-bundle.crt + GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ + SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt ENV \ ENV=/etc/profile \ PATH=/root/.nix-profile/bin:/root/.nix-profile/sbin:/bin:/sbin:/usr/bin:/usr/sbin \ - GIT_SSL_CAINFO=/root/.nix-profile/etc/ca-bundle.crt \ - SSL_CERT_FILE=/root/.nix-profile/etc/ca-bundle.crt + GIT_SSL_CAINFO=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt \ + SSL_CERT_FILE=/root/.nix-profile/etc/ssl/certs/ca-bundle.crt diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index b0dc71fab..f985c5b01 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -7,7 +7,6 @@ $version = "@PACKAGE_VERSION@"; $binDir = $ENV{"NIX_BIN_DIR"} || "@bindir@"; $libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@libexecdir@"; $stateDir = $ENV{"NIX_STATE_DIR"} || "@localstatedir@/nix"; -$manifestDir = $ENV{"NIX_MANIFESTS_DIR"} || "@localstatedir@/nix/manifests"; $logDir = $ENV{"NIX_LOG_DIR"} || "@localstatedir@/log/nix"; $confDir = $ENV{"NIX_CONF_DIR"} || "@sysconfdir@/nix"; $storeDir = $ENV{"NIX_STORE_DIR"} || "@storedir@"; diff --git a/perl/lib/Nix/GeneratePatches.pm b/perl/lib/Nix/GeneratePatches.pm deleted file mode 100644 index 612c8a3a1..000000000 --- a/perl/lib/Nix/GeneratePatches.pm +++ /dev/null @@ -1,340 +0,0 @@ -package Nix::GeneratePatches; - -use strict; -use File::Temp qw(tempdir); -use File::stat; -use Nix::Config; -use Nix::Manifest; - -our @ISA = qw(Exporter); -our @EXPORT = qw(generatePatches propagatePatches copyPatches); - - -# Some patch generations options. - -# Max size of NAR archives to generate patches for. -my $maxNarSize = $ENV{"NIX_MAX_NAR_SIZE"}; -$maxNarSize = 160 * 1024 * 1024 if !defined $maxNarSize; - -# If patch is bigger than this fraction of full archive, reject. -my $maxPatchFraction = $ENV{"NIX_PATCH_FRACTION"}; -$maxPatchFraction = 0.60 if !defined $maxPatchFraction; - -my $timeLimit = $ENV{"NIX_BSDIFF_TIME_LIMIT"}; -$timeLimit = 180 if !defined $timeLimit; - -my $hashAlgo = "sha256"; - - -sub findOutputPaths { - my $narFiles = shift; - - my %outPaths; - - foreach my $p (keys %{$narFiles}) { - - # Ignore derivations. - next if ($p =~ /\.drv$/); - - # Ignore builders (too much ambiguity -- they're all called - # `builder.sh'). - next if ($p =~ /\.sh$/); - next if ($p =~ /\.patch$/); - - # Don't bother including tar files etc. - next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/); - - $outPaths{$p} = 1; - } - - return %outPaths; -} - - -sub getNameVersion { - my $p = shift; - $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; - my $name = $1; - my $version = $2; - return undef unless defined $name && defined $version; - $name =~ s/^-//; - $version =~ s/^-//; - return ($name, $version); -} - - -# A quick hack to get a measure of the `distance' between two -# versions: it's just the position of the first character that differs -# (or 999 if they are the same). -sub versionDiff { - my $s = shift; - my $t = shift; - my $i; - return 999 if $s eq $t; - for ($i = 0; $i < length $s; $i++) { - return $i if $i >= length $t or - substr($s, $i, 1) ne substr($t, $i, 1); - } - return $i; -} - - -sub getNarBz2 { - my $narPath = shift; - my $narFiles = shift; - my $storePath = shift; - - my $narFileList = $$narFiles{$storePath}; - die "missing path $storePath" unless defined $narFileList; - - my $narFile = @{$narFileList}[0]; - die unless defined $narFile; - - $narFile->{url} =~ /\/([^\/]+)$/; - die unless defined $1; - return "$narPath/$1"; -} - - -sub containsPatch { - my $patches = shift; - my $storePath = shift; - my $basePath = shift; - my $patchList = $$patches{$storePath}; - return 0 if !defined $patchList; - my $found = 0; - foreach my $patch (@{$patchList}) { - # !!! baseHash might differ - return 1 if $patch->{basePath} eq $basePath; - } - return 0; -} - - -sub generatePatches { - my ($srcNarFiles, $dstNarFiles, $srcPatches, $dstPatches, $narPath, $patchesPath, $patchesURL, $tmpDir) = @_; - - my %srcOutPaths = findOutputPaths $srcNarFiles; - my %dstOutPaths = findOutputPaths $dstNarFiles; - - # For each output path in the destination, see if we need to / can - # create a patch. - - print STDERR "creating patches...\n"; - - foreach my $p (keys %dstOutPaths) { - - # If exactly the same path already exists in the source, skip it. - next if defined $srcOutPaths{$p}; - - print " $p\n"; - - # If not, then we should find the paths in the source that are - # `most' likely to be present on a system that wants to - # install this path. - - (my $name, my $version) = getNameVersion $p; - next unless defined $name && defined $version; - - my @closest = (); - my $closestVersion; - my $minDist = -1; # actually, larger means closer - - # Find all source paths with the same name. - - foreach my $q (keys %srcOutPaths) { - (my $name2, my $version2) = getNameVersion $q; - next unless defined $name2 && defined $version2; - - if ($name eq $name2) { - - my $srcSystem = @{$$dstNarFiles{$p}}[0]->{system}; - my $dstSystem = @{$$srcNarFiles{$q}}[0]->{system}; - if (defined $srcSystem && defined $dstSystem && $srcSystem ne $dstSystem) { - print " SKIPPING $q due to different systems ($srcSystem vs. $dstSystem)\n"; - next; - } - - # If the sizes differ too much, then skip. This - # disambiguates between, e.g., a real component and a - # wrapper component (cf. Firefox in Nixpkgs). - my $srcSize = @{$$srcNarFiles{$q}}[0]->{size}; - my $dstSize = @{$$dstNarFiles{$p}}[0]->{size}; - my $ratio = $srcSize / $dstSize; - $ratio = 1 / $ratio if $ratio < 1; - # print " SIZE $srcSize $dstSize $ratio $q\n"; - - if ($ratio >= 3) { - print " SKIPPING $q due to size ratio $ratio ($srcSize vs. $dstSize)\n"; - next; - } - - # If there are multiple matching names, include the - # ones with the closest version numbers. - my $dist = versionDiff $version, $version2; - if ($dist > $minDist) { - $minDist = $dist; - @closest = ($q); - $closestVersion = $version2; - } elsif ($dist == $minDist) { - push @closest, $q; - } - } - } - - if (scalar(@closest) == 0) { - print " NO BASE: $p\n"; - next; - } - - foreach my $closest (@closest) { - - # Generate a patch between $closest and $p. - print STDERR " $p <- $closest\n"; - - # If the patch already exists, skip it. - if (containsPatch($srcPatches, $p, $closest) || - containsPatch($dstPatches, $p, $closest)) - { - print " skipping, already exists\n"; - next; - } - - my $srcNarBz2 = getNarBz2 $narPath, $srcNarFiles, $closest; - my $dstNarBz2 = getNarBz2 $narPath, $dstNarFiles, $p; - - if (! -f $srcNarBz2) { - warn "patch source archive $srcNarBz2 is missing\n"; - next; - } - - system("$Nix::Config::bzip2 -d < $srcNarBz2 > $tmpDir/A") == 0 - or die "cannot unpack $srcNarBz2"; - - if (stat("$tmpDir/A")->size >= $maxNarSize) { - print " skipping, source is too large\n"; - next; - } - - system("$Nix::Config::bzip2 -d < $dstNarBz2 > $tmpDir/B") == 0 - or die "cannot unpack $dstNarBz2"; - - if (stat("$tmpDir/B")->size >= $maxNarSize) { - print " skipping, destination is too large\n"; - next; - } - - my $time1 = time(); - my $res = system("ulimit -t $timeLimit; $Nix::Config::libexecDir/nix/bsdiff $tmpDir/A $tmpDir/B $tmpDir/DIFF"); - my $time2 = time(); - if ($res) { - warn "binary diff computation aborted after ", $time2 - $time1, " seconds\n"; - next; - } - - my $baseHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/A` or die; - chomp $baseHash; - - my $narHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/B` or die; - chomp $narHash; - - my $narDiffHash = `$Nix::Config::binDir/nix-hash --flat --type $hashAlgo --base32 $tmpDir/DIFF` or die; - chomp $narDiffHash; - - my $narDiffSize = stat("$tmpDir/DIFF")->size; - my $dstNarBz2Size = stat($dstNarBz2)->size; - - print " size $narDiffSize; full size $dstNarBz2Size; ", $time2 - $time1, " seconds\n"; - - if ($narDiffSize >= $dstNarBz2Size) { - print " rejecting; patch bigger than full archive\n"; - next; - } - - if ($narDiffSize / $dstNarBz2Size >= $maxPatchFraction) { - print " rejecting; patch too large relative to full archive\n"; - next; - } - - my $finalName = "$narDiffHash.nar-bsdiff"; - - if (-e "$patchesPath/$finalName") { - print " not copying, already exists\n"; - } - - else { - system("cp '$tmpDir/DIFF' '$patchesPath/$finalName.tmp'") == 0 - or die "cannot copy diff"; - rename("$patchesPath/$finalName.tmp", "$patchesPath/$finalName") - or die "cannot rename $patchesPath/$finalName.tmp"; - } - - # Add the patch to the manifest. - addPatch $dstPatches, $p, - { url => "$patchesURL/$finalName", hash => "$hashAlgo:$narDiffHash" - , size => $narDiffSize, basePath => $closest, baseHash => "$hashAlgo:$baseHash" - , narHash => "$hashAlgo:$narHash", patchType => "nar-bsdiff" - }; - } - } -} - - -# Propagate useful patches from $srcPatches to $dstPatches. A patch -# is useful if it produces either paths in the $dstNarFiles or paths -# that can be used as the base for other useful patches. -sub propagatePatches { - my ($srcPatches, $dstNarFiles, $dstPatches) = @_; - - print STDERR "propagating patches...\n"; - - my $changed; - do { - # !!! we repeat this to reach the transitive closure; inefficient - $changed = 0; - - print STDERR "loop\n"; - - my %dstBasePaths; - foreach my $q (keys %{$dstPatches}) { - foreach my $patch (@{$$dstPatches{$q}}) { - $dstBasePaths{$patch->{basePath}} = 1; - } - } - - foreach my $p (keys %{$srcPatches}) { - my $patchList = $$srcPatches{$p}; - - my $include = 0; - - # Is path $p included in the destination? If so, include - # patches that produce it. - $include = 1 if defined $$dstNarFiles{$p}; - - # Is path $p a path that serves as a base for paths in the - # destination? If so, include patches that produce it. - # !!! check baseHash - $include = 1 if defined $dstBasePaths{$p}; - - if ($include) { - foreach my $patch (@{$patchList}) { - $changed = 1 if addPatch $dstPatches, $p, $patch; - } - } - - } - - } while $changed; -} - - -# Add all new patches in $srcPatches to $dstPatches. -sub copyPatches { - my ($srcPatches, $dstPatches) = @_; - foreach my $p (keys %{$srcPatches}) { - addPatch $dstPatches, $p, $_ foreach @{$$srcPatches{$p}}; - } -} - - -return 1; diff --git a/perl/lib/Nix/Manifest.pm b/perl/lib/Nix/Manifest.pm index 428decf09..0da376761 100644 --- a/perl/lib/Nix/Manifest.pm +++ b/perl/lib/Nix/Manifest.pm @@ -13,7 +13,7 @@ use Nix::Config; use Nix::Store; our @ISA = qw(Exporter); -our @EXPORT = qw(readManifest writeManifest updateManifestDB addPatch deleteOldManifests parseNARInfo fingerprintPath); +our @EXPORT = qw(readManifest writeManifest addPatch parseNARInfo fingerprintPath); sub addNAR { @@ -228,172 +228,6 @@ sub writeManifest { } -sub updateManifestDB { - my $manifestDir = $Nix::Config::manifestDir; - - my @manifests = glob "$manifestDir/*.nixmanifest"; - return undef if scalar @manifests == 0; - - mkpath($manifestDir); - - unlink "$manifestDir/cache.sqlite"; # remove obsolete cache - my $dbPath = "$manifestDir/cache-v2.sqlite"; - - # Open/create the database. - our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "") - or die "cannot open database ‘$dbPath’"; - $dbh->{RaiseError} = 1; - $dbh->{PrintError} = 0; - - $dbh->do("pragma foreign_keys = on"); - $dbh->do("pragma synchronous = off"); # we can always reproduce the cache - $dbh->do("pragma journal_mode = truncate"); - - # Initialise the database schema, if necessary. - $dbh->do(<<EOF); - create table if not exists Manifests ( - id integer primary key autoincrement not null, - path text unique not null, - timestamp integer not null - ); -EOF - - $dbh->do(<<EOF); - create table if not exists NARs ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - url text not null, - compressionType text not null, - hash text, - size integer, - narHash text, - narSize integer, - refs text, - deriver text, - system text, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists NARs_storePath on NARs(storePath)"); - - $dbh->do(<<EOF); - create table if not exists Patches ( - id integer primary key autoincrement not null, - manifest integer not null, - storePath text not null, - basePath text not null, - baseHash text not null, - url text not null, - hash text, - size integer, - narHash text, - narSize integer, - patchType text not null, - foreign key (manifest) references Manifests(id) on delete cascade - ); -EOF - - $dbh->do("create index if not exists Patches_storePath on Patches(storePath)"); - - # Acquire an exclusive lock to ensure that only one process - # updates the DB at the same time. This isn't really necessary, - # but it prevents work duplication and lock contention in SQLite. - my $lockFile = "$manifestDir/cache.lock"; - open MAINLOCK, ">>$lockFile" or die "unable to acquire lock ‘$lockFile’: $!\n"; - flock(MAINLOCK, LOCK_EX) or die; - - our $insertNAR = $dbh->prepare( - "insert into NARs(manifest, storePath, url, compressionType, hash, size, narHash, " . - "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die; - - our $insertPatch = $dbh->prepare( - "insert into Patches(manifest, storePath, basePath, baseHash, url, hash, " . - "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); - - $dbh->begin_work; - - # Read each manifest in $manifestDir and add it to the database, - # unless we've already done so on a previous run. - my %seen; - - for my $manifestLink (@manifests) { - my $manifest = Cwd::abs_path($manifestLink); - next unless -f $manifest; - my $timestamp = lstat($manifest)->mtime; - $seen{$manifest} = 1; - - next if scalar @{$dbh->selectcol_arrayref( - "select 1 from Manifests where path = ? and timestamp = ?", - {}, $manifest, $timestamp)} == 1; - - print STDERR "caching $manifest...\n"; - - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - - $dbh->do("insert into Manifests(path, timestamp) values (?, ?)", - {}, $manifest, $timestamp); - - our $id = $dbh->last_insert_id("", "", "", ""); - - sub addNARToDB { - my ($storePath, $narFile) = @_; - $insertNAR->execute( - $id, $storePath, $narFile->{url}, $narFile->{compressionType}, $narFile->{hash}, - $narFile->{size}, $narFile->{narHash}, $narFile->{narSize}, $narFile->{references}, - $narFile->{deriver}, $narFile->{system}); - }; - - sub addPatchToDB { - my ($storePath, $patch) = @_; - $insertPatch->execute( - $id, $storePath, $patch->{basePath}, $patch->{baseHash}, $patch->{url}, - $patch->{hash}, $patch->{size}, $patch->{narHash}, $patch->{narSize}, - $patch->{patchType}); - }; - - my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB); - - if ($version < 3) { - die "you have an old-style or corrupt manifest ‘$manifestLink’; please delete it\n"; - } - if ($version >= 10) { - die "manifest ‘$manifestLink’ is too new; please delete it or upgrade Nix\n"; - } - } - - # Removed cached information for removed manifests from the DB. - foreach my $manifest (@{$dbh->selectcol_arrayref("select path from Manifests")}) { - next if defined $seen{$manifest}; - $dbh->do("delete from Manifests where path = ?", {}, $manifest); - } - - $dbh->commit; - - close MAINLOCK; - - return $dbh; -} - - -# Delete all old manifests downloaded from a given URL. -sub deleteOldManifests { - my ($url, $curUrlFile) = @_; - for my $urlFile (glob "$Nix::Config::manifestDir/*.url") { - next if defined $curUrlFile && $urlFile eq $curUrlFile; - open URL, "<$urlFile" or die; - my $url2 = <URL>; - chomp $url2; - close URL; - next unless $url eq $url2; - my $base = $urlFile; $base =~ s/.url$//; - unlink "${base}.url"; - unlink "${base}.nixmanifest"; - } -} - - # Return a fingerprint of a store path to be used in binary cache # signatures. It contains the store path, the base-32 SHA-256 hash of # the contents of the path, and the references. diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index bb322875d..6723ca38c 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -70,8 +70,7 @@ int isValidPath(char * path) SV * queryReferences(char * path) PPCODE: try { - PathSet paths; - store()->queryReferences(path, paths); + PathSet paths = store()->queryPathInfo(path)->references; for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); } catch (Error & e) { @@ -82,7 +81,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - Hash hash = store()->queryPathHash(path); + auto hash = store()->queryPathInfo(path)->narHash; string s = "sha256:" + printHash32(hash); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -93,7 +92,7 @@ SV * queryPathHash(char * path) SV * queryDeriver(char * path) PPCODE: try { - Path deriver = store()->queryDeriver(path); + auto deriver = store()->queryPathInfo(path)->deriver; if (deriver == "") XSRETURN_UNDEF; XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0))); } catch (Error & e) { @@ -104,17 +103,17 @@ SV * queryDeriver(char * path) SV * queryPathInfo(char * path, int base32) PPCODE: try { - ValidPathInfo info = store()->queryPathInfo(path); - if (info.deriver == "") + auto info = store()->queryPathInfo(path); + if (info->deriver == "") XPUSHs(&PL_sv_undef); else - XPUSHs(sv_2mortal(newSVpv(info.deriver.c_str(), 0))); - string s = "sha256:" + (base32 ? printHash32(info.narHash) : printHash(info.narHash)); + XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0))); + string s = "sha256:" + (base32 ? printHash32(info->narHash) : printHash(info->narHash)); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); - mXPUSHi(info.registrationTime); - mXPUSHi(info.narSize); + mXPUSHi(info->registrationTime); + mXPUSHi(info->narSize); AV * arr = newAV(); - for (PathSet::iterator i = info.references.begin(); i != info.references.end(); ++i) + for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i) av_push(arr, newSVpv(i->c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { diff --git a/perl/local.mk b/perl/local.mk index ed49e3e66..5b43c4b71 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -1,7 +1,6 @@ nix_perl_sources := \ $(d)/lib/Nix/Store.pm \ $(d)/lib/Nix/Manifest.pm \ - $(d)/lib/Nix/GeneratePatches.pm \ $(d)/lib/Nix/SSH.pm \ $(d)/lib/Nix/CopyClosure.pm \ $(d)/lib/Nix/Config.pm.in \ diff --git a/release.nix b/release.nix index c602b3da6..6fab352b2 100644 --- a/release.nix +++ b/release.nix @@ -25,7 +25,7 @@ let buildInputs = [ curl bison flex perl libxml2 libxslt bzip2 xz - dblatex (dblatex.tex or tetex) nukeReferences pkgconfig sqlite libsodium + pkgconfig sqlite libsodium docbook5 docbook5_xsl ] ++ lib.optional (!lib.inNixShell) git; @@ -57,35 +57,28 @@ let preDist = '' make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk - - make doc/manual/manual.pdf - cp doc/manual/manual.pdf $out/manual.pdf - - # The PDF containes filenames of included graphics (see - # http://www.tug.org/pipermail/pdftex/2007-August/007290.html). - # This causes a retained dependency on dblatex, which Hydra - # doesn't like (the output of the tarball job is distributed - # to Windows and Macs, so there should be no Linux binaries - # in the closure). - nuke-refs $out/manual.pdf - echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products - echo "doc-pdf manual $out/manual.pdf" >> $out/nix-support/hydra-build-products ''; }; build = pkgs.lib.genAttrs systems (system: - # FIXME: temporarily use a different branch for the Darwin build. - with import (if system == "x86_64-darwin" then <nixpkgs-darwin> else <nixpkgs>) { inherit system; }; + with import <nixpkgs> { inherit system; }; releaseTools.nixBuild { name = "nix"; src = tarball; buildInputs = - [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc ] + [ curl perl bzip2 xz openssl pkgconfig sqlite boehmgc + + (aws-sdk-cpp.override { + apis = ["s3"]; + customMemoryManagement = false; + }) + + ] ++ lib.optional stdenv.isLinux libsodium; configureFlags = '' @@ -113,7 +106,7 @@ let binaryTarball = pkgs.lib.genAttrs systems (system: # FIXME: temporarily use a different branch for the Darwin build. - with import (if system == "x86_64-darwin" then <nixpkgs-darwin> else <nixpkgs>) { inherit system; }; + with import <nixpkgs> { inherit system; }; let toplevel = builtins.getAttr system jobs.build; diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index ee214b930..bd8b44025 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -53,7 +53,7 @@ sub all { $_ || return 0 for @_; 1 } # Initialisation. my $loadIncreased = 0; -my ($localSystem, $maxSilentTime, $printBuildTrace, $buildTimeout) = @ARGV; +my ($localSystem, $maxSilentTime, $buildTimeout) = @ARGV; my $currentLoad = $ENV{"NIX_CURRENT_LOAD"} // "/run/nix/current-load"; my $conf = $ENV{"NIX_REMOTE_SYSTEMS"} // "@sysconfdir@/nix/machines"; @@ -223,9 +223,6 @@ my @inputs = split /\s/, readline(STDIN); my @outputs = split /\s/, readline(STDIN); -print STDERR "@ build-remote $drvPath $hostName\n" if $printBuildTrace; - - my $maybeSign = ""; $maybeSign = "--sign" if -e "$Nix::Config::confDir/signing-key.sec"; @@ -259,13 +256,11 @@ close UPLOADLOCK; # Perform the build. print STDERR "building ‘$drvPath’ on ‘$hostName’\n"; -print STDERR "@ build-remote-start $drvPath $hostName\n" if $printBuildTrace; writeInt(6, $to) or die; # == cmdBuildPaths writeStrings([$drvPath], $to); writeInt($maxSilentTime, $to); writeInt($buildTimeout, $to); my $res = readInt($from); -print STDERR "@ build-remote-done $drvPath $hostName\n" if $printBuildTrace; if ($res != 0) { my $msg = decode("utf-8", readString($from)); print STDERR "error: $msg on ‘$hostName’\n"; diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in deleted file mode 100755 index ffc49f8ff..000000000 --- a/scripts/download-using-manifests.pl.in +++ /dev/null @@ -1,376 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::Config; -use Nix::Manifest; -use Nix::Store; -use Nix::Utils; -use POSIX qw(strftime); - -STDOUT->autoflush(1); -binmode STDERR, ":encoding(utf8)"; - -my $logFile = "$Nix::Config::logDir/downloads"; - -# For queries, skip expensive calls to nix-hash etc. We're just -# estimating the expected download size. -my $fast = 1; - -my $curl = "$Nix::Config::curl --fail --location"; - - -# Open the manifest cache and update it if necessary. -my $dbh = updateManifestDB(); -exit 0 unless defined $dbh; # exit if there are no manifests -print "\n"; - - -# $hashCache->{$algo}->{$path} yields the $algo-hash of $path. -my $hashCache; - - -sub parseHash { - my $hash = shift; - if ($hash =~ /^(.+):(.+)$/) { - return ($1, $2); - } else { - return ("md5", $hash); - } -} - - -# Compute the most efficient sequence of downloads to produce the -# given path. -sub computeSmallestDownload { - my $targetPath = shift; - - # Build a graph of all store paths that might contribute to the - # construction of $targetPath, and the special node "start". The - # edges are either patch operations, or downloads of full NAR - # files. The latter edges only occur between "start" and a store - # path. - my %graph; - - $graph{"start"} = {d => 0, pred => undef, edges => []}; - - my @queue = (); - my $queueFront = 0; - my %done; - - sub addNode { - my $graph = shift; - my $u = shift; - $$graph{$u} = {d => 999999999999, pred => undef, edges => []} - unless defined $$graph{$u}; - } - - sub addEdge { - my $graph = shift; - my $u = shift; - my $v = shift; - my $w = shift; - my $type = shift; - my $info = shift; - addNode $graph, $u; - push @{$$graph{$u}->{edges}}, - {weight => $w, start => $u, end => $v, type => $type, info => $info}; - my $n = scalar @{$$graph{$u}->{edges}}; - } - - push @queue, $targetPath; - - while ($queueFront < scalar @queue) { - my $u = $queue[$queueFront++]; - next if defined $done{$u}; - $done{$u} = 1; - - addNode \%graph, $u; - - # If the path already exists, it has distance 0 from the - # "start" node. - if (isValidPath($u)) { - addEdge \%graph, "start", $u, 0, "present", undef; - } - - else { - - # Add patch edges. - my $patchList = $dbh->selectall_arrayref( - "select * from Patches where storePath = ?", - { Slice => {} }, $u); - - foreach my $patch (@{$patchList}) { - if (isValidPath($patch->{basePath})) { - my ($baseHashAlgo, $baseHash) = parseHash $patch->{baseHash}; - - my $hash = $hashCache->{$baseHashAlgo}->{$patch->{basePath}}; - if (!defined $hash) { - $hash = $fast && $baseHashAlgo eq "sha256" - ? queryPathHash($patch->{basePath}) - : hashPath($baseHashAlgo, $baseHashAlgo ne "md5", $patch->{basePath}); - $hash =~ s/.*://; - $hashCache->{$baseHashAlgo}->{$patch->{basePath}} = $hash; - } - - next if $hash ne $baseHash; - } - push @queue, $patch->{basePath}; - addEdge \%graph, $patch->{basePath}, $u, $patch->{size}, "patch", $patch; - } - - # Add NAR file edges to the start node. - my $narFileList = $dbh->selectall_arrayref( - "select * from NARs where storePath = ?", - { Slice => {} }, $u); - - foreach my $narFile (@{$narFileList}) { - # !!! how to handle files whose size is not known in advance? - # For now, assume some arbitrary size (1 GB). - # This has the side-effect of preferring non-Hydra downloads. - addEdge \%graph, "start", $u, ($narFile->{size} || 1000000000), "narfile", $narFile; - } - } - } - - - # Run Dijkstra's shortest path algorithm to determine the shortest - # sequence of download and/or patch actions that will produce - # $targetPath. - - my @todo = keys %graph; - - while (scalar @todo > 0) { - - # Remove the closest element from the todo list. - # !!! inefficient, use a priority queue - @todo = sort { -($graph{$a}->{d} <=> $graph{$b}->{d}) } @todo; - my $u = pop @todo; - - my $u_ = $graph{$u}; - - foreach my $edge (@{$u_->{edges}}) { - my $v_ = $graph{$edge->{end}}; - if ($v_->{d} > $u_->{d} + $edge->{weight}) { - $v_->{d} = $u_->{d} + $edge->{weight}; - # Store the edge; to edge->start is actually the - # predecessor. - $v_->{pred} = $edge; - } - } - } - - - # Retrieve the shortest path from "start" to $targetPath. - my @path = (); - my $cur = $targetPath; - return () unless defined $graph{$targetPath}->{pred}; - while ($cur ne "start") { - push @path, $graph{$cur}->{pred}; - $cur = $graph{$cur}->{pred}->{start}; - } - - return @path; -} - - -# Parse the arguments. - -if ($ARGV[0] eq "--query") { - - while (<STDIN>) { - chomp; - my ($cmd, @args) = split " ", $_; - - if ($cmd eq "have") { - foreach my $storePath (@args) { - print "$storePath\n" if scalar @{$dbh->selectcol_arrayref("select 1 from NARs where storePath = ?", {}, $storePath)} > 0; - } - print "\n"; - } - - elsif ($cmd eq "info") { - foreach my $storePath (@args) { - - my $infos = $dbh->selectall_arrayref( - "select * from NARs where storePath = ?", - { Slice => {} }, $storePath); - - next unless scalar @{$infos} > 0; - my $info = @{$infos}[0]; - - print "$storePath\n"; - print "$info->{deriver}\n"; - my @references = split " ", $info->{refs}; - print scalar @references, "\n"; - print "$_\n" foreach @references; - - my @path = computeSmallestDownload $storePath; - - my $downloadSize = 0; - while (scalar @path > 0) { - my $edge = pop @path; - my $u = $edge->{start}; - my $v = $edge->{end}; - if ($edge->{type} eq "patch") { - $downloadSize += $edge->{info}->{size} || 0; - } - elsif ($edge->{type} eq "narfile") { - $downloadSize += $edge->{info}->{size} || 0; - } - } - - print "$downloadSize\n"; - - my $narSize = $info->{narSize} || 0; - print "$narSize\n"; - } - - print "\n"; - } - - else { die "unknown command ‘$cmd’"; } - } - - exit 0; -} - -elsif ($ARGV[0] ne "--substitute") { - die; -} - - -die unless scalar @ARGV == 3; -my $targetPath = $ARGV[1]; -my $destPath = $ARGV[2]; -$fast = 0; - - -# Create a temporary directory. -my $tmpDir = mkTempDir("nix-download"); - -my $tmpNar = "$tmpDir/nar"; -my $tmpNar2 = "$tmpDir/nar2"; - - -open LOGFILE, ">>$logFile" or die "cannot open log file $logFile"; - -my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); -print LOGFILE "$$ get $targetPath $date\n"; - -print STDERR "\n*** Trying to download/patch ‘$targetPath’\n"; - - -# Compute the shortest path. -my @path = computeSmallestDownload $targetPath; -die "don't know how to produce $targetPath\n" if scalar @path == 0; - - -# We don't need the manifest anymore, so close it as an optimisation: -# if we still have SQLite locks blocking other processes (we -# shouldn't), this gets rid of them. -$dbh->disconnect; - - -# Traverse the shortest path, perform the actions described by the -# edges. -my $curStep = 1; -my $maxStep = scalar @path; - -my $finalNarHash; - -while (scalar @path > 0) { - my $edge = pop @path; - my $u = $edge->{start}; - my $v = $edge->{end}; - - print STDERR "\n*** Step $curStep/$maxStep: "; - - if ($edge->{type} eq "present") { - print STDERR "using already present path ‘$v’\n"; - print LOGFILE "$$ present $v\n"; - - if ($curStep < $maxStep) { - # Since this is not the last step, the path will be used - # as a base to one or more patches. So turn the base path - # into a NAR archive, to which we can apply the patch. - print STDERR " packing base path...\n"; - system("$Nix::Config::binDir/nix-store --dump $v > $tmpNar") == 0 - or die "cannot dump ‘$v’"; - } - } - - elsif ($edge->{type} eq "patch") { - my $patch = $edge->{info}; - print STDERR "applying patch ‘$patch->{url}’ to ‘$u’ to create ‘$v’\n"; - - print LOGFILE "$$ patch $patch->{url} $patch->{size} $patch->{baseHash} $u $v\n"; - - # Download the patch. - print STDERR " downloading patch...\n"; - my $patchPath = "$tmpDir/patch"; - checkURL $patch->{url}; - system("$curl '$patch->{url}' -o $patchPath") == 0 - or die "cannot download patch ‘$patch->{url}’\n"; - - # Apply the patch to the NAR archive produced in step 1 (for - # the already present path) or a later step (for patch sequences). - print STDERR " applying patch...\n"; - system("$Nix::Config::libexecDir/nix/bspatch $tmpNar $tmpNar2 $patchPath") == 0 - or die "cannot apply patch ‘$patchPath’ to $tmpNar\n"; - - if ($curStep < $maxStep) { - # The archive will be used as the base of the next patch. - rename "$tmpNar2", "$tmpNar" or die "cannot rename NAR archive: $!"; - } else { - # This was the last patch. Unpack the final NAR archive - # into the target path. - print STDERR " unpacking patched archive...\n"; - system("$Nix::Config::binDir/nix-store --restore $destPath < $tmpNar2") == 0 - or die "cannot unpack $tmpNar2 to ‘$v’\n"; - } - - $finalNarHash = $patch->{narHash}; - } - - elsif ($edge->{type} eq "narfile") { - my $narFile = $edge->{info}; - print STDERR "downloading ‘$narFile->{url}’ to ‘$v’\n"; - - my $size = $narFile->{size} || -1; - print LOGFILE "$$ narfile $narFile->{url} $size $v\n"; - - checkURL $narFile->{url}; - - my $decompressor = - $narFile->{compressionType} eq "bzip2" ? "| $Nix::Config::bzip2 -d" : - $narFile->{compressionType} eq "xz" ? "| $Nix::Config::xz -d" : - $narFile->{compressionType} eq "none" ? "" : - die "unknown compression type ‘$narFile->{compressionType}’"; - - if ($curStep < $maxStep) { - # The archive will be used a base to a patch. - system("$curl '$narFile->{url}' $decompressor > $tmpNar") == 0 - or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n"; - } else { - # Unpack the archive to the target path. - system("$curl '$narFile->{url}' $decompressor | $Nix::Config::binDir/nix-store --restore '$destPath'") == 0 - or die "cannot download and unpack ‘$narFile->{url}’ to ‘$v’\n"; - } - - $finalNarHash = $narFile->{narHash}; - } - - $curStep++; -} - - -# Tell Nix about the expected hash so it can verify it. -die "cannot check integrity of the downloaded path since its hash is not known\n" - unless defined $finalNarHash; -print "$finalNarHash\n"; - - -print STDERR "\n"; -print LOGFILE "$$ success\n"; -close LOGFILE; diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index bef5cd4f1..509acc41f 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#!/bin/sh set -e @@ -49,7 +49,10 @@ for i in $(cd $self/store >/dev/null && echo *); do fi if ! [ -e "$dest/store/$i" ]; then cp -Rp "$self/store/$i" "$i_tmp" + chmod -R a-w "$i_tmp" + chmod +w "$i_tmp" mv "$i_tmp" "$dest/store/$i" + chmod -w "$dest/store/$i" fi done echo "" >&2 diff --git a/scripts/local.mk b/scripts/local.mk index cdac56bf1..13b13a86b 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -2,26 +2,18 @@ nix_bin_scripts := \ $(d)/nix-build \ $(d)/nix-channel \ $(d)/nix-copy-closure \ - $(d)/nix-generate-patches \ $(d)/nix-install-package \ - $(d)/nix-pull \ $(d)/nix-push bin-scripts += $(nix_bin_scripts) -nix_substituters := \ - $(d)/copy-from-other-stores.pl \ - $(d)/download-from-binary-cache.pl \ - $(d)/download-using-manifests.pl - nix_noinst_scripts := \ $(d)/build-remote.pl \ $(d)/find-runtime-roots.pl \ $(d)/resolve-system-dependencies.pl \ $(d)/nix-http-export.cgi \ $(d)/nix-profile.sh \ - $(d)/nix-reduce-build \ - $(nix_substituters) + $(d)/nix-reduce-build noinst-scripts += $(nix_noinst_scripts) @@ -31,7 +23,6 @@ $(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) $(eval $(call install-program-in, $(d)/find-runtime-roots.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/build-remote.pl, $(libexecdir)/nix)) $(eval $(call install-program-in, $(d)/resolve-system-dependencies.pl, $(libexecdir)/nix)) -$(foreach prog, $(nix_substituters), $(eval $(call install-program-in, $(prog), $(libexecdir)/nix/substituters))) $(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) clean-files += $(nix_bin_scripts) $(nix_noinst_scripts) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index b93e5ab13..78a69c94e 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -110,13 +110,6 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--log-type") { - $n++; - die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; - push @instArgs, ($arg, $ARGV[$n]); - push @buildArgs, ($arg, $ARGV[$n]); - } - elsif ($arg eq "--option") { die "$0: ‘$arg’ requires two arguments\n" unless $n + 2 < scalar @ARGV; push @instArgs, ($arg, $ARGV[$n + 1], $ARGV[$n + 2]); @@ -124,7 +117,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--log-type" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { + elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { $n++; die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV; push @buildArgs, ($arg, $ARGV[$n]); diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index 5191b5855..65084ff1f 100755 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in @@ -12,8 +12,6 @@ binmode STDERR, ":encoding(utf8)"; Nix::Config::readConfig; -my $manifestDir = $Nix::Config::manifestDir; - # Turn on caching in nix-prefetch-url. my $channelCache = "$Nix::Config::stateDir/channel-cache"; @@ -75,7 +73,6 @@ sub removeChannel { my ($name) = @_; readChannels; my $url = $channels{$name}; - deleteOldManifests($url . "/MANIFEST", undef) if defined $url; delete $channels{$name}; writeChannels; @@ -84,8 +81,7 @@ sub removeChannel { } -# Fetch Nix expressions and pull manifests from the subscribed -# channels. +# Fetch Nix expressions and binary cache URLs from the subscribed channels. sub update { my @channelNames = @_; @@ -97,7 +93,6 @@ sub update { next if scalar @channelNames > 0 && ! grep { $_ eq $name } @{channelNames}; my $url = $channels{$name}; - my $origUrl = "$url/MANIFEST"; # We want to download the url to a file to see if it's a tarball while also checking if we # got redirected in the process, so that we can grab the various parts of a nix channel @@ -132,22 +127,8 @@ sub update { if ($ret != 0) { # Check if the channel advertises a binary cache. my $binaryCacheURL = `$Nix::Config::curl --silent '$url'/binary-cache-url`; - my $getManifest = ($Nix::Config::config{"force-manifest"} // "false") eq "true"; - if ($? == 0 && $binaryCacheURL ne "") { - $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; "; - deleteOldManifests($origUrl, undef); - } else { - $getManifest = 1; - } - - if ($getManifest) { - # No binary cache, so pull the channel manifest. - mkdir $manifestDir, 0755 unless -e $manifestDir; - die "$0: you do not have write permission to ‘$manifestDir’!\n" unless -W $manifestDir; - $ENV{'NIX_ORIG_URL'} = $origUrl; - system("$Nix::Config::binDir/nix-pull", "--skip-wrong-store", "$url/MANIFEST") == 0 - or die "cannot pull manifest from ‘$url’\n"; - } + $extraAttrs .= "binaryCacheURL = \"$binaryCacheURL\"; " + if $? == 0 && $binaryCacheURL ne ""; # Download the channel tarball. my $fullURL = "$url/nixexprs.tar.xz"; diff --git a/scripts/nix-generate-patches.in b/scripts/nix-generate-patches.in deleted file mode 100755 index 0a29c0548..000000000 --- a/scripts/nix-generate-patches.in +++ /dev/null @@ -1,51 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use strict; -use Nix::Manifest; -use Nix::GeneratePatches; -use Nix::Utils; - -if (scalar @ARGV != 5) { - print STDERR <<EOF; -Usage: nix-generate-patches NAR-DIR PATCH-DIR PATCH-URI OLD-MANIFEST NEW-MANIFEST - -This command generates binary patches between NAR files listed in -OLD-MANIFEST and NEW-MANIFEST. The patches are written to the -directory PATCH-DIR, and the prefix PATCH-URI is used to generate URIs -for the patches. The patches are added to NEW-MANIFEST. All NARs are -required to exist in NAR-DIR. Patches are generated between -succeeding versions of packages with the same name. -EOF - exit 1; -} - -my $narPath = $ARGV[0]; -my $patchesPath = $ARGV[1]; -my $patchesURL = $ARGV[2]; -my $srcManifest = $ARGV[3]; -my $dstManifest = $ARGV[4]; - -my (%srcNarFiles, %srcLocalPaths, %srcPatches); -readManifest $srcManifest, \%srcNarFiles, \%srcPatches; - -my (%dstNarFiles, %dstLocalPaths, %dstPatches); -readManifest $dstManifest, \%dstNarFiles, \%dstPatches; - -my $tmpDir = mkTempDir("nix-generate-patches"); - -generatePatches \%srcNarFiles, \%dstNarFiles, \%srcPatches, \%dstPatches, - $narPath, $patchesPath, $patchesURL, $tmpDir; - -propagatePatches \%srcPatches, \%dstNarFiles, \%dstPatches; - -# Optionally add all new patches to the manifest in $NIX_ALL_PATCHES. -my $allPatchesFile = $ENV{"NIX_ALL_PATCHES"}; -if (defined $allPatchesFile) { - my (%dummy, %allPatches); - readManifest("$patchesPath/all-patches", \%dummy, \%allPatches) - if -f $allPatchesFile; - copyPatches \%dstPatches, \%allPatches; - writeManifest($allPatchesFile, {}, \%allPatches, 0); -} - -writeManifest $dstManifest, \%dstNarFiles, \%dstPatches; diff --git a/scripts/nix-install-package.in b/scripts/nix-install-package.in index b442c708b..ba349774a 100755 --- a/scripts/nix-install-package.in +++ b/scripts/nix-install-package.in @@ -89,7 +89,7 @@ my $pathRE = "(?: \/ [\/A-Za-z0-9\+\-\.\_\?\=]* )"; # store path. We'll let nix-env do that. $contents =~ - / ^ \s* (\S+) \s+ ($Nix::Utils::urlRE) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x + / ^ \s* (\S+) \s+ (\S+) \s+ ($nameRE) \s+ ($systemRE) \s+ ($pathRE) \s+ ($pathRE) ( \s+ ($Nix::Utils::urlRE) )? /x or barf "invalid package contents"; my $version = $1; my $manifestURL = $2; @@ -111,25 +111,9 @@ if ($interactive) { } -if (defined $binaryCacheURL) { +die "$0: package does not supply a binary cache\n" unless defined $binaryCacheURL; - push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL; - -} else { - - # Store the manifest in the temporary directory so that we don't - # pollute /nix/var/nix/manifests. This also requires that we - # don't use the Nix daemon (because otherwise - # download-using-manifests won't see our NIX_MANIFESTS_DIRS - # environment variable). - $ENV{NIX_MANIFESTS_DIR} = $tmpDir; - $ENV{NIX_REMOTE} = ""; - - print "\nPulling manifests...\n"; - system("$Nix::Config::binDir/nix-pull", $manifestURL) == 0 - or barf "nix-pull failed: $?"; - -} +push @extraNixEnvArgs, "--option", "extra-binary-caches", $binaryCacheURL; print "\nInstalling package...\n"; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 6616b12b0..5e01de951 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,24 +1,71 @@ -if [ -n "$HOME" ]; then - NIX_LINK="$HOME/.nix-profile" - - # Set the default profile. - if ! [ -L "$NIX_LINK" ]; then - echo "creating $NIX_LINK" >&2 - _NIX_DEF_LINK=@localstatedir@/nix/profiles/default - @coreutils@/ln -s "$_NIX_DEF_LINK" "$NIX_LINK" +if [ -n "$HOME" ] && [ -n "$USER" ]; then + __savedpath="$PATH" + export PATH=@coreutils@ + + # Set up the per-user profile. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix + + : ${NIX_LINK:=$HOME/.nix-profile} + + : ${NIX_USER_PROFILE_DIR:=@localstatedir@/nix/profiles/per-user/$USER} + + mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" + + if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then + echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 fi - export PATH=$NIX_LINK/bin:$NIX_LINK/sbin:$PATH + if [ -w "$HOME" ]; then + if ! [ -L "$NIX_LINK" ]; then + echo "Nix: creating $NIX_LINK" >&2 + if [ "$USER" != root ]; then + if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then + echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 + fi + else + # Root installs in the system-wide profile by default. + ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" + fi + fi - # Subscribe the user to the Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" + # Subscribe the user to the unstable Nixpkgs channel by default. + if [ ! -e "$HOME/.nix-channels" ]; then + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" + fi + + # Create the per-user garbage collector roots directory. + __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" + mkdir -m 0755 -p "$__user_gcroots" + if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then + echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 + fi + unset __user_gcroots + + # Set up a default Nix expression from which to install stuff. + __nix_defexpr="$HOME"/.nix-defexpr + [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" + mkdir -m 0755 -p "$__nix_defexpr" + if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then + ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root + fi + unset __nix_defexpr fi # Append ~/.nix-defexpr/channels/nixpkgs to $NIX_PATH so that # <nixpkgs> paths work when the user has fetched the Nixpkgs # channel. - export NIX_PATH=${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs + export NIX_PATH="${NIX_PATH:+$NIX_PATH:}nixpkgs=$HOME/.nix-defexpr/channels/nixpkgs" + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + export NIX_USER_PROFILE_DIR + export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + + for i in $NIX_PROFILES; do + if [ -d "$i/lib/aspell" ]; then + export ASPELL_CONF="dict-dir $i/lib/aspell" + fi + done # Set $SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -34,4 +81,7 @@ if [ -n "$HOME" ]; then elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # old cacert in Nix profile export SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" fi + + export PATH="$NIX_LINK/bin:$NIX_LINK/sbin:$__savedpath" + unset __savedpath fi diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in deleted file mode 100755 index 995b50935..000000000 --- a/scripts/nix-pull.in +++ /dev/null @@ -1,102 +0,0 @@ -#! @perl@ -w @perlFlags@ - -use utf8; -use strict; -use Nix::Config; -use Nix::Manifest; - -binmode STDERR, ":encoding(utf8)"; - -my $manifestDir = $Nix::Config::manifestDir; - - -# Prevent access problems in shared-stored installations. -umask 0022; - - -# Create the manifests directory if it doesn't exist. -if (! -e $manifestDir) { - mkdir $manifestDir, 0755 or die "cannot create directory ‘$manifestDir’"; -} - - -# Make sure that the manifests directory is scanned for GC roots. -my $gcRootsDir = "$Nix::Config::stateDir/gcroots"; -my $manifestDirLink = "$gcRootsDir/manifests"; -if (! -l $manifestDirLink) { - symlink($manifestDir, $manifestDirLink) or die "cannot create symlink ‘$manifestDirLink’"; -} - - -# Process the URLs specified on the command line. - -sub downloadFile { - my $url = shift; - $ENV{"PRINT_PATH"} = 1; - $ENV{"QUIET"} = 1; - my ($dummy, $path) = `$Nix::Config::binDir/nix-prefetch-url '$url'`; - die "cannot fetch ‘$url’" if $? != 0; - die "nix-prefetch-url did not return a path" unless defined $path; - chomp $path; - return $path; -} - -sub processURL { - my $url = shift; - - $url =~ s/\/$//; - - my $manifest; - - my $origUrl = $ENV{'NIX_ORIG_URL'} || $url; - - # First see if a bzipped manifest is available. - if (system("$Nix::Config::curl --fail --silent --location --head '$url'.bz2 > /dev/null") == 0) { - print "fetching list of Nix archives at ‘$url.bz2’...\n"; - $manifest = downloadFile "$url.bz2"; - } - - # Otherwise, just get the uncompressed manifest. - else { - print "fetching list of Nix archives at ‘$url’...\n"; - $manifest = downloadFile $url; - } - - my $baseName = "unnamed"; - if ($url =~ /\/([^\/]+)\/[^\/]+$/) { # get the forelast component - $baseName = $1; - } - - my $hash = `$Nix::Config::binDir/nix-hash --flat '$manifest'` - or die "cannot hash ‘$manifest’"; - chomp $hash; - - my $urlFile = "$manifestDir/$baseName-$hash.url"; - open URL, ">$urlFile" or die "cannot create ‘$urlFile’"; - print URL $origUrl; - close URL; - - my $finalPath = "$manifestDir/$baseName-$hash.nixmanifest"; - - unlink $finalPath if -e $finalPath; - - symlink("$manifest", "$finalPath") - or die "cannot link ‘$finalPath’ to ‘$manifest’"; - - deleteOldManifests($origUrl, $urlFile); -} - -while (@ARGV) { - my $url = shift @ARGV; - if ($url eq "--help") { - exec "man nix-pull" or die; - } elsif ($url eq "--skip-wrong-store") { - # No-op, no longer supported. - } else { - processURL $url; - } -} - - -# Update the cache. -updateManifestDB(); diff --git a/src/bsdiff-4.3/bsdiff.1 b/src/bsdiff-4.3/bsdiff.1 deleted file mode 100644 index ead6c4deb..000000000 --- a/src/bsdiff-4.3/bsdiff.1 +++ /dev/null @@ -1,63 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.1,v 1.1 2005/08/06 01:59:05 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSDIFF 1 -.Os FreeBSD -.Sh NAME -.Nm bsdiff -.Nd generate a patch between two binary files -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -compares -.Ao Ar oldfile Ac -to -.Ao Ar newfile Ac -and writes to -.Ao Ar patchfile Ac -a binary patch suitable for use by bspatch(1). -When -.Ao Ar oldfile Ac -and -.Ao Ar newfile Ac -are two versions of an executable program, the -patches produced are on average a factor of five smaller -than those produced by any other binary patch tool known -to the author. -.Pp -.Nm -uses memory equal to 17 times the size of -.Ao Ar oldfile Ac , -and requires -an absolute minimum working set size of 8 times the size of oldfile. -.Sh SEE ALSO -.Xr bspatch 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c deleted file mode 100644 index 374ed038f..000000000 --- a/src/bsdiff-4.3/bsdiff.c +++ /dev/null @@ -1,405 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05 cperciva Exp $"); -#endif - -#include <sys/types.h> - -#include <bzlib.h> -#include <err.h> -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#define MIN(x,y) (((x)<(y)) ? (x) : (y)) - -static void split(off_t *I,off_t *V,off_t start,off_t len,off_t h) -{ - off_t i,j,k,x,tmp,jj,kk; - - if(len<16) { - for(k=start;k<start+len;k+=j) { - j=1;x=V[I[k]+h]; - for(i=1;k+i<start+len;i++) { - if(V[I[k+i]+h]<x) { - x=V[I[k+i]+h]; - j=0; - }; - if(V[I[k+i]+h]==x) { - tmp=I[k+j];I[k+j]=I[k+i];I[k+i]=tmp; - j++; - }; - }; - for(i=0;i<j;i++) V[I[k+i]]=k+j-1; - if(j==1) I[k]=-1; - }; - return; - }; - - x=V[I[start+len/2]+h]; - jj=0;kk=0; - for(i=start;i<start+len;i++) { - if(V[I[i]+h]<x) jj++; - if(V[I[i]+h]==x) kk++; - }; - jj+=start;kk+=jj; - - i=start;j=0;k=0; - while(i<jj) { - if(V[I[i]+h]<x) { - i++; - } else if(V[I[i]+h]==x) { - tmp=I[i];I[i]=I[jj+j];I[jj+j]=tmp; - j++; - } else { - tmp=I[i];I[i]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - while(jj+j<kk) { - if(V[I[jj+j]+h]==x) { - j++; - } else { - tmp=I[jj+j];I[jj+j]=I[kk+k];I[kk+k]=tmp; - k++; - }; - }; - - if(jj>start) split(I,V,start,jj-start,h); - - for(i=0;i<kk-jj;i++) V[I[jj+i]]=kk-1; - if(jj==kk-1) I[jj]=-1; - - if(start+len>kk) split(I,V,kk,start+len-kk,h); -} - -static void qsufsort(off_t *I,off_t *V,u_char *old,off_t oldsize) -{ - off_t buckets[256]; - off_t i,h,len; - - for(i=0;i<256;i++) buckets[i]=0; - for(i=0;i<oldsize;i++) buckets[old[i]]++; - for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; - for(i=255;i>0;i--) buckets[i]=buckets[i-1]; - buckets[0]=0; - - for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; - I[0]=oldsize; - for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; - V[oldsize]=0; - for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; - I[0]=-1; - - for(h=1;I[0]!=-(oldsize+1);h+=h) { - len=0; - for(i=0;i<oldsize+1;) { - if(I[i]<0) { - len-=I[i]; - i-=I[i]; - } else { - if(len) I[i-len]=-len; - len=V[I[i]]+1-i; - split(I,V,i,len,h); - i+=len; - len=0; - }; - }; - if(len) I[i-len]=-len; - }; - - for(i=0;i<oldsize+1;i++) I[V[i]]=i; -} - -static off_t matchlen(u_char *old,off_t oldsize,u_char *new,off_t newsize) -{ - off_t i; - - for(i=0;(i<oldsize)&&(i<newsize);i++) - if(old[i]!=new[i]) break; - - return i; -} - -static off_t search(off_t *I,u_char *old,off_t oldsize, - u_char *new,off_t newsize,off_t st,off_t en,off_t *pos) -{ - off_t x,y; - - if(en-st<2) { - x=matchlen(old+I[st],oldsize-I[st],new,newsize); - y=matchlen(old+I[en],oldsize-I[en],new,newsize); - - if(x>y) { - *pos=I[st]; - return x; - } else { - *pos=I[en]; - return y; - } - }; - - x=st+(en-st)/2; - if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { - return search(I,old,oldsize,new,newsize,x,en,pos); - } else { - return search(I,old,oldsize,new,newsize,st,x,pos); - }; -} - -static void offtout(off_t x,u_char *buf) -{ - off_t y; - - if(x<0) y=-x; else y=x; - - buf[0]=y%256;y-=buf[0]; - y=y/256;buf[1]=y%256;y-=buf[1]; - y=y/256;buf[2]=y%256;y-=buf[2]; - y=y/256;buf[3]=y%256;y-=buf[3]; - y=y/256;buf[4]=y%256;y-=buf[4]; - y=y/256;buf[5]=y%256;y-=buf[5]; - y=y/256;buf[6]=y%256;y-=buf[6]; - y=y/256;buf[7]=y%256; - - if(x<0) buf[7]|=0x80; -} - -int main(int argc,char *argv[]) -{ - int fd; - u_char *old,*new; - off_t oldsize,newsize; - off_t *I,*V; - off_t scan,pos,len; - off_t lastscan,lastpos,lastoffset; - off_t oldscore,scsc; - off_t s,Sf,lenf,Sb,lenb; - off_t overlap,Ss,lens; - off_t i; - off_t dblen,eblen; - u_char *db,*eb; - u_char buf[8]; - u_char header[32]; - FILE * pf; - BZFILE * pfbz2; - int bz2err; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Allocate oldsize+1 bytes instead of oldsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - - if(((I=malloc((oldsize+1)*sizeof(off_t)))==NULL) || - ((V=malloc((oldsize+1)*sizeof(off_t)))==NULL)) err(1,NULL); - - qsufsort(I,V,old,oldsize); - - free(V); - - /* Allocate newsize+1 bytes instead of newsize bytes to ensure - that we never try to malloc(0) and get a NULL pointer */ - if(((fd=open(argv[2],O_RDONLY,0))<0) || - ((newsize=lseek(fd,0,SEEK_END))==-1) || - ((new=malloc(newsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,new,newsize)!=newsize) || - (close(fd)==-1)) err(1,"%s",argv[2]); - - if(((db=malloc(newsize+1))==NULL) || - ((eb=malloc(newsize+1))==NULL)) err(1,NULL); - dblen=0; - eblen=0; - - /* Create the patch file */ - if ((pf = fopen(argv[3], "w")) == NULL) - err(1, "%s", argv[3]); - - /* Header is - 0 8 "BSDIFF40" - 8 8 length of bzip2ed ctrl block - 16 8 length of bzip2ed diff block - 24 8 length of new file */ - /* File is - 0 32 Header - 32 ?? Bzip2ed ctrl block - ?? ?? Bzip2ed diff block - ?? ?? Bzip2ed extra block */ - memcpy(header,"BSDIFF40",8); - offtout(0, header + 8); - offtout(0, header + 16); - offtout(newsize, header + 24); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - - /* Compute the differences, writing ctrl as we go */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - scan=0;len=0; - lastscan=0;lastpos=0;lastoffset=0; - while(scan<newsize) { - oldscore=0; - - for(scsc=scan+=len;scan<newsize;scan++) { - len=search(I,old,oldsize,new+scan,newsize-scan, - 0,oldsize,&pos); - if (len > 64 * 1024) break; - - for(;scsc<scan+len;scsc++) - if((scsc+lastoffset<oldsize) && - (old[scsc+lastoffset] == new[scsc])) - oldscore++; - - if(((len==oldscore) && (len!=0)) || - (len>oldscore+8)) break; - - if((scan+lastoffset<oldsize) && - (old[scan+lastoffset] == new[scan])) - oldscore--; - }; - - if((len!=oldscore) || (scan==newsize)) { - s=0;Sf=0;lenf=0; - for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { - if(old[lastpos+i]==new[lastscan+i]) s++; - i++; - if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; - }; - - lenb=0; - if(scan<newsize) { - s=0;Sb=0; - for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { - if(old[pos-i]==new[scan-i]) s++; - if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; - }; - }; - - if(lastscan+lenf>scan-lenb) { - overlap=(lastscan+lenf)-(scan-lenb); - s=0;Ss=0;lens=0; - for(i=0;i<overlap;i++) { - if(new[lastscan+lenf-overlap+i]== - old[lastpos+lenf-overlap+i]) s++; - if(new[scan-lenb+i]== - old[pos-lenb+i]) s--; - if(s>Ss) { Ss=s; lens=i+1; }; - }; - - lenf+=lens-overlap; - lenb-=lens; - }; - - for(i=0;i<lenf;i++) - db[dblen+i]=new[lastscan+i]-old[lastpos+i]; - for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) - eb[eblen+i]=new[lastscan+lenf+i]; - - dblen+=lenf; - eblen+=(scan-lenb)-(lastscan+lenf); - - offtout(lenf,buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((scan-lenb)-(lastscan+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - offtout((pos-lenb)-(lastpos+lenf),buf); - BZ2_bzWrite(&bz2err, pfbz2, buf, 8); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - - lastscan=scan-lenb; - lastpos=pos-lenb; - lastoffset=pos-scan; - }; - }; - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed ctrl data */ - if ((len = ftello(pf)) == -1) - err(1, "ftello"); - offtout(len-32, header + 8); - - /* Write compressed diff data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, db, dblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Compute size of compressed diff data */ - if ((newsize = ftello(pf)) == -1) - err(1, "ftello"); - offtout(newsize - len, header + 16); - - /* Write compressed extra data */ - if ((pfbz2 = BZ2_bzWriteOpen(&bz2err, pf, 9, 0, 0)) == NULL) - errx(1, "BZ2_bzWriteOpen, bz2err = %d", bz2err); - BZ2_bzWrite(&bz2err, pfbz2, eb, eblen); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWrite, bz2err = %d", bz2err); - BZ2_bzWriteClose(&bz2err, pfbz2, 0, NULL, NULL); - if (bz2err != BZ_OK) - errx(1, "BZ2_bzWriteClose, bz2err = %d", bz2err); - - /* Seek to the beginning, write the header, and close the file */ - if (fseeko(pf, 0, SEEK_SET)) - err(1, "fseeko"); - if (fwrite(header, 32, 1, pf) != 1) - err(1, "fwrite(%s)", argv[3]); - if (fclose(pf)) - err(1, "fclose"); - - /* Free the memory we used */ - free(db); - free(eb); - free(I); - free(old); - free(new); - - return 0; -} diff --git a/src/bsdiff-4.3/bspatch.1 b/src/bsdiff-4.3/bspatch.1 deleted file mode 100644 index 82a2781aa..000000000 --- a/src/bsdiff-4.3/bspatch.1 +++ /dev/null @@ -1,59 +0,0 @@ -.\"- -.\" Copyright 2003-2005 Colin Percival -.\" All rights reserved -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted providing that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -.\" WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.\" POSSIBILITY OF SUCH DAMAGE. -.\" -.\" $FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.1,v 1.1 2005/08/06 01:59:06 cperciva Exp $ -.\" -.Dd May 18, 2003 -.Dt BSPATCH 1 -.Os FreeBSD -.Sh NAME -.Nm bspatch -.Nd apply a patch built with bsdiff(1) -.Sh SYNOPSIS -.Nm -.Ao Ar oldfile Ac Ao Ar newfile Ac Ao Ar patchfile Ac -.Sh DESCRIPTION -.Nm -generates -.Ao Ar newfile Ac -from -.Ao Ar oldfile Ac -and -.Ao Ar patchfile Ac -where -.Ao Ar patchfile Ac -is a binary patch built by bsdiff(1). -.Pp -.Nm -uses memory equal to the size of -.Ao Ar oldfile Ac -plus the size of -.Ao Ar newfile Ac , -but can tolerate a very small working set without a dramatic loss -of performance. -.Sh SEE ALSO -.Xr bsdiff 1 -.Sh AUTHORS -.An Colin Percival Aq cperciva@freebsd.org diff --git a/src/bsdiff-4.3/bspatch.c b/src/bsdiff-4.3/bspatch.c deleted file mode 100644 index f9d33ddd6..000000000 --- a/src/bsdiff-4.3/bspatch.c +++ /dev/null @@ -1,224 +0,0 @@ -/*- - * Copyright 2003-2005 Colin Percival - * All rights reserved - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted providing that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#if 0 -__FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:06 cperciva Exp $"); -#endif - -#include <bzlib.h> -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <err.h> -#include <errno.h> -#include <unistd.h> -#include <fcntl.h> -#include <sys/types.h> - -static off_t offtin(u_char *buf) -{ - off_t y; - - y=buf[7]&0x7F; - y=y*256;y+=buf[6]; - y=y*256;y+=buf[5]; - y=y*256;y+=buf[4]; - y=y*256;y+=buf[3]; - y=y*256;y+=buf[2]; - y=y*256;y+=buf[1]; - y=y*256;y+=buf[0]; - - if(buf[7]&0x80) y=-y; - - return y; -} - - -void writeFull(const char * name, int fd, - const unsigned char * buf, size_t count) -{ - while (count) { - ssize_t res = write(fd, (char *) buf, count); - if (res == -1) { - if (errno == EINTR) continue; - err(1,"writing to %s",name); - } - count -= res; - buf += res; - } -} - - -int main(int argc,char * argv[]) -{ - FILE * f, * cpf, * dpf, * epf; - BZFILE * cpfbz2, * dpfbz2, * epfbz2; - int cbz2err, dbz2err, ebz2err; - int fd; - ssize_t oldsize,newsize; - ssize_t bzctrllen,bzdatalen; - u_char header[32],buf[8]; - u_char *old, *new; - off_t oldpos,newpos; - off_t ctrl[3]; - off_t lenread; - off_t i; - - if(argc!=4) errx(1,"usage: %s oldfile newfile patchfile\n",argv[0]); - - /* Open patch file */ - if ((f = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - - /* - File format: - 0 8 "BSDIFF40" - 8 8 X - 16 8 Y - 24 8 sizeof(newfile) - 32 X bzip2(control block) - 32+X Y bzip2(diff block) - 32+X+Y ??? bzip2(extra block) - with control block a set of triples (x,y,z) meaning "add x bytes - from oldfile to x bytes from the diff block; copy y bytes from the - extra block; seek forwards in oldfile by z bytes". - */ - - /* Read header */ - if (fread(header, 1, 32, f) < 32) { - if (feof(f)) - errx(1, "Corrupt patch\n"); - err(1, "fread(%s)", argv[3]); - } - - /* Check for appropriate magic */ - if (memcmp(header, "BSDIFF40", 8) != 0) - errx(1, "Corrupt patch\n"); - - /* Read lengths from header */ - bzctrllen=offtin(header+8); - bzdatalen=offtin(header+16); - newsize=offtin(header+24); - if((bzctrllen<0) || (bzdatalen<0) || (newsize<0)) - errx(1,"Corrupt patch\n"); - - /* Close patch file and re-open it via libbzip2 at the right places */ - if (fclose(f)) - err(1, "fclose(%s)", argv[3]); - if ((cpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(cpf, 32, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)32); - if ((cpfbz2 = BZ2_bzReadOpen(&cbz2err, cpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", cbz2err); - if ((dpf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(dpf, 32 + bzctrllen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen)); - if ((dpfbz2 = BZ2_bzReadOpen(&dbz2err, dpf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", dbz2err); - if ((epf = fopen(argv[3], "r")) == NULL) - err(1, "fopen(%s)", argv[3]); - if (fseeko(epf, 32 + bzctrllen + bzdatalen, SEEK_SET)) - err(1, "fseeko(%s, %lld)", argv[3], - (long long)(32 + bzctrllen + bzdatalen)); - if ((epfbz2 = BZ2_bzReadOpen(&ebz2err, epf, 0, 0, NULL, 0)) == NULL) - errx(1, "BZ2_bzReadOpen, bz2err = %d", ebz2err); - - if(((fd=open(argv[1],O_RDONLY,0))<0) || - ((oldsize=lseek(fd,0,SEEK_END))==-1) || - ((old=malloc(oldsize+1))==NULL) || - (lseek(fd,0,SEEK_SET)!=0) || - (read(fd,old,oldsize)!=oldsize) || - (close(fd)==-1)) err(1,"%s",argv[1]); - if((new=malloc(newsize+1))==NULL) err(1,NULL); - - oldpos=0;newpos=0; - while(newpos<newsize) { - /* Read control data */ - for(i=0;i<=2;i++) { - lenread = BZ2_bzRead(&cbz2err, cpfbz2, buf, 8); - if ((lenread < 8) || ((cbz2err != BZ_OK) && - (cbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - ctrl[i]=offtin(buf); - }; - - /* Sanity-check */ - if(newpos+ctrl[0]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read diff string */ - lenread = BZ2_bzRead(&dbz2err, dpfbz2, new + newpos, ctrl[0]); - if ((lenread < ctrl[0]) || - ((dbz2err != BZ_OK) && (dbz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Add old data to diff string */ - for(i=0;i<ctrl[0];i++) - if((oldpos+i>=0) && (oldpos+i<oldsize)) - new[newpos+i]+=old[oldpos+i]; - - /* Adjust pointers */ - newpos+=ctrl[0]; - oldpos+=ctrl[0]; - - /* Sanity-check */ - if(newpos+ctrl[1]>newsize) - errx(1,"Corrupt patch\n"); - - /* Read extra string */ - lenread = BZ2_bzRead(&ebz2err, epfbz2, new + newpos, ctrl[1]); - if ((lenread < ctrl[1]) || - ((ebz2err != BZ_OK) && (ebz2err != BZ_STREAM_END))) - errx(1, "Corrupt patch\n"); - - /* Adjust pointers */ - newpos+=ctrl[1]; - oldpos+=ctrl[2]; - }; - - /* Clean up the bzip2 reads */ - BZ2_bzReadClose(&cbz2err, cpfbz2); - BZ2_bzReadClose(&dbz2err, dpfbz2); - BZ2_bzReadClose(&ebz2err, epfbz2); - if (fclose(cpf) || fclose(dpf) || fclose(epf)) - err(1, "fclose(%s)", argv[3]); - - /* Write the new file */ - if((fd=open(argv[2],O_CREAT|O_TRUNC|O_WRONLY,0666))<0) - err(1,"%s",argv[2]); - writeFull(argv[2], fd, new, newsize); - if(close(fd)==-1) - err(1,"%s",argv[2]); - - free(new); - free(old); - - return 0; -} diff --git a/src/bsdiff-4.3/compat-include/err.h b/src/bsdiff-4.3/compat-include/err.h deleted file mode 100644 index a851ded6f..000000000 --- a/src/bsdiff-4.3/compat-include/err.h +++ /dev/null @@ -1,12 +0,0 @@ -/* Simulate BSD's <err.h> functionality. */ - -#ifndef COMPAT_ERR_H_INCLUDED -#define COMPAT_ERR_H_INCLUDED 1 - -#include <stdio.h> -#include <stdlib.h> - -#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) -#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0) - -#endif diff --git a/src/bsdiff-4.3/local.mk b/src/bsdiff-4.3/local.mk deleted file mode 100644 index c957ceab0..000000000 --- a/src/bsdiff-4.3/local.mk +++ /dev/null @@ -1,11 +0,0 @@ -programs += bsdiff bspatch - -bsdiff_DIR := $(d) -bsdiff_SOURCES := $(d)/bsdiff.c -bsdiff_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bsdiff_INSTALL_DIR = $(libexecdir)/nix - -bspatch_DIR := $(d) -bspatch_SOURCES := $(d)/bspatch.c -bspatch_LDFLAGS = -lbz2 $(bsddiff_compat_include) -bspatch_INSTALL_DIR = $(libexecdir)/nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8ce2f3dfa..5a6428ca6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -5,6 +5,7 @@ #include "derivations.hh" #include "globals.hh" #include "eval-inline.hh" +#include "download.hh" #include <algorithm> #include <cstring> @@ -238,12 +239,38 @@ void initGC() /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const string & in) +static Strings parseNixPath(const string & s) { - string marker = "\001//"; - auto res = tokenizeString<Strings>(replaceStrings(in, "://", marker), ":"); - for (auto & s : res) - s = replaceStrings(s, marker, "://"); + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + return res; } @@ -278,7 +305,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) /* Initialise the Nix expression search path. */ Strings paths = parseNixPath(getEnv("NIX_PATH", "")); - for (auto & i : _searchPath) addToSearchPath(i, true); + for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : paths) addToSearchPath(i); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); @@ -301,11 +328,15 @@ Path EvalState::checkSourcePath(const Path & path_) if (!restricted) return path_; /* Resolve symlinks. */ + debug(format("checking access to ‘%s’") % path_); Path path = canonPath(path_, true); - for (auto & i : searchPath) - if (path == i.second || isInDir(path, i.second)) + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + if (path == r.second || isInDir(path, r.second)) return path; + } /* To support import-from-derivation, allow access to anything in the store. FIXME: only allow access to paths that have been @@ -610,7 +641,7 @@ void EvalState::evalFile(const Path & path, Value & v) return; } - startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path2); + Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path2); Expr * e = parseExprFromFile(checkSourcePath(path2)); try { eval(e, v); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 40e05712b..80e369f2d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -26,9 +26,9 @@ typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, struct PrimOp { PrimOpFun fun; - unsigned int arity; + size_t arity; Symbol name; - PrimOp(PrimOpFun fun, unsigned int arity, Symbol name) + PrimOp(PrimOpFun fun, size_t arity, Symbol name) : fun(fun), arity(arity), name(name) { } }; @@ -56,7 +56,8 @@ typedef std::map<Path, Path> SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); -typedef list<std::pair<string, Path> > SearchPath; +typedef std::pair<std::string, std::string> SearchPathElem; +typedef std::list<SearchPathElem> SearchPath; /* Initialise the Boehm GC, if applicable. */ @@ -98,12 +99,14 @@ private: SearchPath searchPath; + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + public: EvalState(const Strings & _searchPath, ref<Store> store); ~EvalState(); - void addToSearchPath(const string & s, bool warn = false); + void addToSearchPath(const string & s); Path checkSourcePath(const Path & path); @@ -125,6 +128,9 @@ public: Path findFile(const string & path); Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); + /* Evaluate an expression to normal form, storing the result in value `v'. */ void eval(Expr * e, Value & v); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 996c2c5f4..b06c539de 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -30,7 +30,7 @@ string DrvInfo::queryOutPath() } -DrvInfo::Outputs DrvInfo::queryOutputs() +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ @@ -55,7 +55,23 @@ DrvInfo::Outputs DrvInfo::queryOutputs() } else outputs["out"] = queryOutPath(); } - return outputs; + if (!onlyOutputsToInstall || !attrs) + return outputs; + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value * outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + const auto errMsg = Error("this derivation has bad ‘meta.outputsToInstall’"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { + if ((*i)->type != tString) throw errMsg; + auto out = outputs.find((*i)->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); + } + return result; } @@ -192,8 +208,8 @@ typedef set<Bindings *> Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', - then put information about it in `drvs' (unless it's already in - `doneExprs'). The result boolean indicates whether it makes sense + then put information about it in `drvs' (unless it's already in `done'). + The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ static bool getDerivation(EvalState & state, Value & v, const string & attrPath, DrvInfos & drvs, Done & done, @@ -274,7 +290,7 @@ static void getDerivations(EvalState & state, Value & vIn, attrs.insert(std::pair<string, Symbol>(i.name, i.name)); for (auto & i : attrs) { - startNest(nest, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); + Activity act(*logger, lvlDebug, format("evaluating attribute ‘%1%’") % i.first); string pathPrefix2 = addToPath(pathPrefix, i.first); Value & v2(*v.attrs->find(i.second)->value); if (combineChannels) @@ -294,8 +310,7 @@ static void getDerivations(EvalState & state, Value & vIn, else if (v.isList()) { for (unsigned int n = 0; n < v.listSize(); ++n) { - startNest(nest, lvlDebug, - format("evaluating list element")); + Activity act(*logger, lvlDebug, "evaluating list element"); string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 365c66c8d..37fcbe829 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -42,7 +42,8 @@ public: string queryDrvPath(); string queryOutPath(); string queryOutputName(); - Outputs queryOutputs(); + /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); Value * queryMeta(const string & name); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 5de9ccc6d..620050a13 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,7 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_CXXFLAGS := -Wno-deprecated-register diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 11dc7bb5c..776e5cb39 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -520,9 +520,10 @@ formal #include <fcntl.h> #include <unistd.h> -#include <eval.hh> -#include <download.hh> -#include <store-api.hh> +#include "eval.hh" +#include "download.hh" +#include "store-api.hh" +#include "primops/fetchgit.hh" namespace nix { @@ -600,7 +601,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) } -void EvalState::addToSearchPath(const string & s, bool warn) +void EvalState::addToSearchPath(const string & s) { size_t pos = s.find('='); string prefix; @@ -612,16 +613,7 @@ void EvalState::addToSearchPath(const string & s, bool warn) path = string(s, pos + 1); } - if (isUri(path)) - path = makeDownloader()->downloadCached(store, path, true); - - path = absPath(path); - if (pathExists(path)) { - debug(format("adding path ‘%1%’ to the search path") % path); - /* Resolve symlinks in the path to support restricted mode. */ - searchPath.push_back(std::pair<string, Path>(prefix, canonPath(path, true))); - } else if (warn) - printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % path); + searchPath.emplace_back(prefix, path); } @@ -634,17 +626,19 @@ Path EvalState::findFile(const string & path) Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) { for (auto & i : searchPath) { - assert(!isUri(i.second)); - Path res; + std::string suffix; if (i.first.empty()) - res = i.second + "/" + path; + suffix = "/" + path; else { - if (path.compare(0, i.first.size(), i.first) != 0 || - (path.size() > i.first.size() && path[i.first.size()] != '/')) + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) continue; - res = i.second + - (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size())); + suffix = path.size() == s ? "" : "/" + string(path, s); } + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } format f = format( @@ -655,4 +649,39 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos } +std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem) +{ + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) return i->second; + + std::pair<bool, std::string> res; + + if (isUri(elem.second)) { + try { + if (hasPrefix(elem.second, "git://") || hasSuffix(elem.second, ".git")) + // FIXME: support specifying revision/branch + res = { true, exportGit(store, elem.second, "master") }; + else + res = { true, makeDownloader()->downloadCached(store, elem.second, true) }; + } catch (DownloadError & e) { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ cannot be downloaded, ignoring") % elem.second); + res = { false, "" }; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) + res = { true, path }; + else { + printMsg(lvlError, format("warning: Nix search path entry ‘%1%’ does not exist, ignoring") % elem.second); + res = { false, "" }; + } + } + + debug(format("resolved search path element ‘%s’ to ‘%s’") % elem.second % res.second); + + searchPathResolved[elem.second] = res; + return res; +} + + } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aaef467c0..c2852629a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -10,6 +10,7 @@ #include "util.hh" #include "value-to-json.hh" #include "value-to-xml.hh" +#include "primops.hh" #include <sys/types.h> #include <sys/stat.h> @@ -123,7 +124,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args env->values[displ++] = attr.value; } - startNest(nest, lvlTalkative, format("evaluating file ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("evaluating file ‘%1%’") % path); Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); @@ -283,7 +284,7 @@ typedef list<Value *> ValueList; static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { - startNest(nest, lvlDebug, "finding dependencies"); + Activity act(*logger, lvlDebug, "finding dependencies"); state.forceAttrs(*args[0], pos); @@ -456,7 +457,7 @@ void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & derivation. */ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) { - startNest(nest, lvlVomit, "evaluating derivation"); + Activity act(*logger, lvlVomit, "evaluating derivation"); state.forceAttrs(*args[0], pos); @@ -493,7 +494,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : *args[0]->attrs) { if (i.name == state.sIgnoreNulls) continue; string key = i.name; - startNest(nest, lvlVomit, format("processing attribute ‘%1%’") % key); + Activity act(*logger, lvlVomit, format("processing attribute ‘%1%’") % key); try { @@ -777,7 +778,6 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - PathSet context; for (unsigned int n = 0; n < args[0]->listSize(); ++n) { Value & v2(*args[0]->listElems()[n]); state.forceAttrs(v2, pos); @@ -790,21 +790,23 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError(format("attribute ‘path’ missing, at %1%") % pos); - string path = state.coerceToPath(pos, *i->value, context); - searchPath.push_back(std::pair<string, Path>(prefix, state.checkSourcePath(path))); - } + PathSet context; + string path = state.coerceToString(pos, *i->value, context, false, false); - string path = state.forceStringNoCtx(*args[1], pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") + % path % e.path % pos); + } - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError(format("cannot find ‘%1%’, since path ‘%2%’ is not valid, at %3%") - % path % e.path % pos); + searchPath.emplace_back(prefix, path); } - mkPath(v, state.findFile(searchPath, path, pos).c_str()); + string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } /* Read a directory (without . or ..) */ @@ -1725,6 +1727,16 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args *************************************************************/ +RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; + + +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +{ + if (!primOps) primOps = new PrimOps; + primOps->emplace_back(name, arity, fun); +} + + void EvalState::createBaseEnv() { baseEnv.up = 0; @@ -1889,6 +1901,10 @@ void EvalState::createBaseEnv() } addConstant("__nixPath", v); + if (RegisterPrimOp::primOps) + for (auto & primOp : *RegisterPrimOp::primOps) + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh new file mode 100644 index 000000000..39d23b04a --- /dev/null +++ b/src/libexpr/primops.hh @@ -0,0 +1,15 @@ +#include "eval.hh" + +#include <tuple> +#include <vector> + +namespace nix { + +struct RegisterPrimOp +{ + typedef std::vector<std::tuple<std::string, size_t, PrimOpFun>> PrimOps; + static PrimOps * primOps; + RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); +}; + +} diff --git a/src/libexpr/primops/fetchgit.cc b/src/libexpr/primops/fetchgit.cc new file mode 100644 index 000000000..bd440c8c6 --- /dev/null +++ b/src/libexpr/primops/fetchgit.cc @@ -0,0 +1,82 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "store-api.hh" + +namespace nix { + +Path exportGit(ref<Store> store, const std::string & uri, const std::string & rev) +{ + if (!isUri(uri)) + throw EvalError(format("‘%s’ is not a valid URI") % uri); + + Path cacheDir = getCacheDir() + "/nix/git"; + + if (!pathExists(cacheDir)) { + createDirs(cacheDir); + runProgram("git", true, { "init", "--bare", cacheDir }); + } + + Activity act(*logger, lvlInfo, format("fetching Git repository ‘%s’") % uri); + + std::string localRef = "pid-" + std::to_string(getpid()); + Path localRefFile = cacheDir + "/refs/heads/" + localRef; + + runProgram("git", true, { "-C", cacheDir, "fetch", uri, rev + ":" + localRef }); + + std::string commitHash = chomp(readFile(localRefFile)); + + unlink(localRefFile.c_str()); + + debug(format("got revision ‘%s’") % commitHash); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, { "-C", cacheDir, "archive", commitHash }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("tar", true, { "x", "-C", tmpDir }, tar); + + return store->addToStore("git-export", tmpDir); +} + +static void prim_fetchgit(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + // FIXME: cut&paste from fetch(). + if (state.restricted) throw Error("‘fetchgit’ is not allowed in restricted mode"); + + std::string url; + std::string rev = "master"; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string name(attr.name); + if (name == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (name == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError(format("unsupported argument ‘%1%’ to ‘fetchgit’, at %3%") % attr.name % attr.pos); + } + + if (url.empty()) + throw EvalError(format("‘url’ argument required, at %1%") % pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + Path storePath = exportGit(state.store, url, rev); + + mkString(v, storePath, PathSet({storePath})); +} + +static RegisterPrimOp r("__fetchgit", 1, prim_fetchgit); + +} diff --git a/src/libexpr/primops/fetchgit.hh b/src/libexpr/primops/fetchgit.hh new file mode 100644 index 000000000..6ffb21a96 --- /dev/null +++ b/src/libexpr/primops/fetchgit.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <string> + +#include "ref.hh" + +namespace nix { + +class Store; + +Path exportGit(ref<Store> store, + const std::string & uri, const std::string & rev); + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 9219f380c..98693d78a 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -18,15 +18,6 @@ MixCommonArgs::MixCommonArgs(const string & programName) verbosity = lvlDebug; }); - mkFlag1(0, "log-type", "type", "set logging format ('pretty', 'flat', 'systemd')", - [](std::string s) { - if (s == "pretty") logType = ltPretty; - else if (s == "escapes") logType = ltEscapes; - else if (s == "flat") logType = ltFlat; - else if (s == "systemd") logType = ltSystemd; - else throw UsageError("unknown log type"); - }); - mkFlag(0, "option", {"name", "value"}, "set a Nix configuration option (overriding nix.conf)", 2, [](Strings ss) { auto name = ss.front(); ss.pop_front(); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index e883967b7..0b6311516 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -24,15 +24,9 @@ namespace nix { -volatile sig_atomic_t blockInt = 0; - - static void sigintHandler(int signo) { - if (!blockInt) { - _isInterrupted = 1; - blockInt = 1; - } + _isInterrupted = 1; } @@ -117,11 +111,7 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif - // FIXME: do we need this? It's not thread-safe. - std::ios::sync_with_stdio(false); - - if (getEnv("IN_SYSTEMD") == "1") - logType = ltSystemd; + logger = makeDefaultLogger(); /* Initialise OpenSSL locking. */ opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); @@ -181,10 +171,7 @@ struct LegacyArgs : public MixCommonArgs : MixCommonArgs(programName), parseArg(parseArg) { mkFlag('Q', "no-build-output", "do not show build output", - &settings.buildVerbosity, lvlVomit); - - mkFlag(0, "print-build-trace", "emit special build trace message", - &settings.printBuildTrace); + &settings.verboseBuild, false); mkFlag('K', "keep-failed", "keep temporary directories of failed builds", &settings.keepFailed); @@ -290,8 +277,7 @@ int handleExceptions(const string & programName, std::function<void()> fun) condition is discharged before we reach printMsg() below, since otherwise it will throw an (uncaught) exception. */ - blockInt = 1; /* ignore further SIGINTs */ - _isInterrupted = 0; + interruptThrown = true; throw; } } catch (Exit & e) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 94f5cbabb..411d10130 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -8,20 +8,20 @@ #include "sync.hh" #include "worker-protocol.hh" #include "nar-accessor.hh" +#include "nar-info-disk-cache.hh" #include <chrono> namespace nix { BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile) + const StoreParams & params) : localStore(localStore) + , compression(get(params, "compression", "xz")) { - if (secretKeyFile != "") { + auto secretKeyFile = get(params, "secret-key", ""); + if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); - publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys); - publicKeys->emplace(secretKey->name, secretKey->toPublicKey()); - } StringSink sink; sink << narVersionMagic1; @@ -40,57 +40,65 @@ void BinaryCacheStore::notImpl() throw Error("operation not implemented for binary cache stores"); } -const BinaryCacheStore::Stats & BinaryCacheStore::getStats() -{ - return stats; -} - Path BinaryCacheStore::narInfoFileFor(const Path & storePath) { assertStorePath(storePath); return storePathToHash(storePath) + ".narinfo"; } -void BinaryCacheStore::addToCache(const ValidPathInfo & info, - const string & nar) +void BinaryCacheStore::addToCache(const ValidPathInfo & info, ref<std::string> nar) { + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) + try { + if (ref != info.path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error(format("cannot add ‘%s’ to the binary cache because the reference ‘%s’ is not valid") + % info.path % ref); + } + auto narInfoFile = narInfoFileFor(info.path); if (fileExists(narInfoFile)) return; - assert(nar.compare(0, narMagic.size(), narMagic) == 0); + assert(nar->compare(0, narMagic.size(), narMagic) == 0); auto narInfo = make_ref<NarInfo>(info); - narInfo->narSize = nar.size(); - narInfo->narHash = hashString(htSHA256, nar); + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); - if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash) + if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); /* Compress the NAR. */ - narInfo->compression = "xz"; + narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); - string narXz = compressXZ(nar); + auto narCompressed = compress(compression, nar); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, narXz); - narInfo->fileSize = narXz.size(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") % narInfo->path % narInfo->narSize - % ((1.0 - (double) narXz.size() / nar.size()) * 100.0) + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); /* Atomically write the NAR file. */ - narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz"; + narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + ""); if (!fileExists(narInfo->url)) { stats.narWrite++; - upsertFile(narInfo->url, narXz); + upsertFile(narInfo->url, *narCompressed); } else stats.narWriteAverted++; - stats.narWriteBytes += nar.size(); - stats.narWriteCompressedBytes += narXz.size(); + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ @@ -98,95 +106,67 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info, upsertFile(narInfoFile, narInfo->to_string()); - { - auto state_(state.lock()); - state_->narInfoCache.upsert(narInfo->path, narInfo); - stats.narInfoCacheSize = state_->narInfoCache.size(); - } - - stats.narInfoWrite++; -} + auto hashPart = storePathToHash(narInfo->path); -NarInfo BinaryCacheStore::readNarInfo(const Path & storePath) -{ { auto state_(state.lock()); - auto res = state_->narInfoCache.get(storePath); - if (res) { - stats.narInfoReadAverted++; - return **res; - } + state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); } - auto narInfoFile = narInfoFileFor(storePath); - auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile); - if (narInfo->path != storePath) - throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath); - - stats.narInfoRead++; - - if (publicKeys) { - if (!narInfo->checkSignature(*publicKeys)) - throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile); - } - - { - auto state_(state.lock()); - state_->narInfoCache.upsert(storePath, narInfo); - stats.narInfoCacheSize = state_->narInfoCache.size(); - } + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); - return *narInfo; + stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPath(const Path & storePath) +bool BinaryCacheStore::isValidPathUncached(const Path & storePath) { - { - auto state_(state.lock()); - auto res = state_->narInfoCache.get(storePath); - if (res) { - stats.narInfoReadAverted++; - return true; - } - } - // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even // though they shouldn't. Not easily fixed. return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink) +void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) { - assert(!sign); + auto info = queryPathInfo(storePath).cast<const NarInfo>(); - auto res = readNarInfo(storePath); + auto nar = getFile(info->url); - auto nar = getFile(res.url); + if (!nar) throw Error(format("file ‘%s’ missing from binary cache") % info->url); stats.narRead++; - stats.narReadCompressedBytes += nar.size(); + stats.narReadCompressedBytes += nar->size(); /* Decompress the NAR. FIXME: would be nice to have the remote side do this. */ - if (res.compression == "none") - ; - else if (res.compression == "xz") - nar = decompressXZ(nar); - else - throw Error(format("unknown NAR compression type ‘%1%’") % nar); + try { + nar = decompress(info->compression, ref<std::string>(nar)); + } catch (UnknownCompressionMethod &) { + throw Error(format("binary cache path ‘%s’ uses unknown compression method ‘%s’") + % storePath % info->compression); + } + + stats.narReadBytes += nar->size(); - stats.narReadBytes += nar.size(); + printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar->size()); - printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size()); + assert(nar->size() % 8 == 0); - assert(nar.size() % 8 == 0); + sink((unsigned char *) nar->c_str(), nar->size()); +} + +void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink) +{ + assert(!sign); + + auto res = queryPathInfo(storePath); - sink((unsigned char *) nar.c_str(), nar.size()); + narFromPath(storePath, sink); // FIXME: check integrity of NAR. - sink << exportMagic << storePath << res.references << res.deriver << 0; + sink << exportMagic << storePath << res->references << res->deriver << 0; } Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source, @@ -224,9 +204,17 @@ struct NopSink : ParseSink { }; -ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath) +std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Path & storePath) { - return ValidPathInfo(readNarInfo(storePath)); + auto narInfoFile = narInfoFileFor(storePath); + auto data = getFile(narInfoFile); + if (!data) return 0; + + auto narInfo = make_ref<NarInfo>(*data, narInfoFile); + + stats.narInfoRead++; + + return std::shared_ptr<NarInfo>(narInfo); } void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, @@ -237,16 +225,16 @@ void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths, if (!localStore) return; for (auto & storePath : paths) { - if (!localStore->isValidPath(storePath)) { + try { + auto info = localStore->queryPathInfo(storePath); + SubstitutablePathInfo sub; + sub.references = info->references; + sub.downloadSize = 0; + sub.narSize = info->narSize; + infos.emplace(storePath, sub); + } catch (InvalidPath &) { left.insert(storePath); - continue; } - ValidPathInfo info = localStore->queryPathInfo(storePath); - SubstitutablePathInfo sub; - sub.references = info.references; - sub.downloadSize = 0; - sub.narSize = info.narSize; - infos.emplace(storePath, sub); } if (settings.useSubstitutes) @@ -276,7 +264,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, info.path = makeFixedOutputPath(recursive, hashAlgo, h, name); if (repair || !isValidPath(info.path)) - addToCache(info, *sink.s); + addToCache(info, sink.s); return info.path; } @@ -291,7 +279,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToCache(info, *sink.s); + addToCache(info, sink.s); } return info.path; @@ -312,16 +300,16 @@ void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode) if (!localStore->isValidPath(storePath)) localStore->ensurePath(storePath); - ValidPathInfo info = localStore->queryPathInfo(storePath); + auto info = localStore->queryPathInfo(storePath); - for (auto & ref : info.references) + for (auto & ref : info->references) if (ref != storePath) ensurePath(ref); StringSink sink; dumpPath(storePath, sink); - addToCache(info, *sink.s); + addToCache(*info, sink.s); } } @@ -419,7 +407,7 @@ Path BinaryCacheStore::importPath(Source & source, std::shared_ptr<FSAccessor> a bool haveSignature = readInt(source) == 1; assert(!haveSignature); - addToCache(info, *tee.data); + addToCache(info, tee.data); auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); if (accessor_) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c99556f33..46a38a1e0 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -3,8 +3,6 @@ #include "crypto.hh" #include "store-api.hh" -#include "lru-cache.hh" -#include "sync.hh" #include "pool.hh" #include <atomic> @@ -18,20 +16,15 @@ class BinaryCacheStore : public Store private: std::unique_ptr<SecretKey> secretKey; - std::unique_ptr<PublicKeys> publicKeys; std::shared_ptr<Store> localStore; - struct State - { - LRUCache<Path, ref<NarInfo>> narInfoCache{32 * 1024}; - }; - - Sync<State> state; + std::string compression; protected: - BinaryCacheStore(std::shared_ptr<Store> localStore, const Path & secretKeyFile); + BinaryCacheStore(std::shared_ptr<Store> localStore, + const StoreParams & params); [[noreturn]] void notImpl(); @@ -39,47 +32,25 @@ protected: virtual void upsertFile(const std::string & path, const std::string & data) = 0; - virtual std::string getFile(const std::string & path) = 0; + /* Return the contents of the specified file, or null if it + doesn't exist. */ + virtual std::shared_ptr<std::string> getFile(const std::string & path) = 0; public: virtual void init(); - struct Stats - { - std::atomic<uint64_t> narInfoRead{0}; - std::atomic<uint64_t> narInfoReadAverted{0}; - std::atomic<uint64_t> narInfoWrite{0}; - std::atomic<uint64_t> narInfoCacheSize{0}; - std::atomic<uint64_t> narRead{0}; - std::atomic<uint64_t> narReadBytes{0}; - std::atomic<uint64_t> narReadCompressedBytes{0}; - std::atomic<uint64_t> narWrite{0}; - std::atomic<uint64_t> narWriteAverted{0}; - std::atomic<uint64_t> narWriteBytes{0}; - std::atomic<uint64_t> narWriteCompressedBytes{0}; - std::atomic<uint64_t> narWriteCompressionTimeMs{0}; - }; - - const Stats & getStats(); - private: - Stats stats; - std::string narMagic; std::string narInfoFileFor(const Path & storePath); - void addToCache(const ValidPathInfo & info, const string & nar); - -protected: - - NarInfo readNarInfo(const Path & storePath); + void addToCache(const ValidPathInfo & info, ref<std::string> nar); public: - bool isValidPath(const Path & path) override; + bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet & paths) override { notImpl(); } @@ -87,18 +58,12 @@ public: PathSet queryAllValidPaths() override { notImpl(); } - ValidPathInfo queryPathInfo(const Path & path) override; - - Hash queryPathHash(const Path & path) override - { notImpl(); } + std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override; void queryReferrers(const Path & path, PathSet & referrers) override { notImpl(); } - Path queryDeriver(const Path & path) override - { return ""; } - PathSet queryValidDerivers(const Path & path) override { return {}; } @@ -124,6 +89,8 @@ public: Path addTextToStore(const string & name, const string & s, const PathSet & references, bool repair = false) override; + void narFromPath(const Path & path, Sink & sink) override; + void exportPath(const Path & path, bool sign, Sink & sink) override; Paths importPaths(bool requireSignature, Source & source, @@ -154,12 +121,6 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override { notImpl(); } - PathSet queryFailedPaths() override - { return {}; } - - void clearFailedPaths(const PathSet & paths) override - { } - void optimiseStore() override { } @@ -168,6 +129,9 @@ public: ref<FSAccessor> getFSAccessor() override; + void addSignatures(const Path & storePath, const StringSet & sigs) + { notImpl(); } + }; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ed4e0f659..65df2eea5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -8,11 +8,14 @@ #include "archive.hh" #include "affinity.hh" #include "builtins.hh" +#include "finally.hh" #include <algorithm> #include <iostream> #include <map> #include <sstream> +#include <thread> +#include <future> #include <limits.h> #include <time.h> @@ -22,6 +25,7 @@ #include <sys/stat.h> #include <sys/utsname.h> #include <sys/select.h> +#include <sys/resource.h> #include <fcntl.h> #include <unistd.h> #include <errno.h> @@ -198,8 +202,6 @@ struct Child time_t timeStarted; }; -typedef map<pid_t, Child> Children; - /* The worker class. */ class Worker @@ -219,7 +221,7 @@ private: WeakGoals wantingToBuild; /* Child processes currently running. */ - Children children; + std::list<Child> children; /* Number of build slots occupied. This includes local builds and substitutions but not remote builds via the build hook. */ @@ -239,6 +241,9 @@ private: /* Last time the goals in `waitingForAWhile' where woken up. */ time_t lastWokenUp; + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + public: /* Set if at least one derivation had a BuildError (i.e. permanent @@ -274,14 +279,14 @@ public: /* Registers a running child process. `inBuildSlot' means that the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, pid_t pid, - const set<int> & fds, bool inBuildSlot, bool respectTimeouts); + void childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts); /* Unregisters a running child process. `wakeSleepers' should be false if there is no sense in waking up goals that are sleeping because they can't run yet (e.g., there is no free build slot, or the hook would still say `postpone'). */ - void childTerminated(pid_t pid, bool wakeSleepers = true); + void childTerminated(GoalPtr goal, bool wakeSleepers = true); /* Put `goal' to sleep until a build slot becomes available (which might be right away). */ @@ -304,6 +309,12 @@ public: void waitForInput(); unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); }; @@ -623,7 +634,6 @@ HookInstance::HookInstance() baseNameOf(buildHook), settings.thisSystem, (format("%1%") % settings.maxSilentTime).str(), - (format("%1%") % settings.printBuildTrace).str(), (format("%1%") % settings.buildTimeout).str() }; @@ -738,6 +748,12 @@ private: /* Number of bytes received from the builder's stdout/stderr. */ unsigned long logSize; + /* The most recent log lines. */ + std::list<std::string> logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + /* Pipe for the builder's standard output/error. */ Pipe builderOut; @@ -863,6 +879,7 @@ private: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; + void flushLine(); /* Return the set of (in)valid paths. */ PathSet checkPathValidity(bool returnValid, bool checkHash); @@ -926,7 +943,7 @@ DerivationGoal::~DerivationGoal() void DerivationGoal::killChild() { if (pid != -1) { - worker.childTerminated(pid); + worker.childTerminated(shared_from_this()); if (buildUser.enabled()) { /* If we're using a build user, then there is a tricky @@ -950,8 +967,6 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut() { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); killChild(); done(BuildResult::TimedOut); } @@ -1038,11 +1053,6 @@ void DerivationGoal::haveDerivation() return; } - /* Check whether any output previously failed to build. If so, - don't bother. */ - for (auto & i : invalidOutputs) - if (pathFailed(i)) return; - /* Reject doing a hash build of anything other than a fixed-output derivation. */ if (buildMode == bmHash) { @@ -1159,7 +1169,7 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ PathSet broken; for (auto & i : outputClosure) { - if (worker.store.pathContentsGood(i)) continue; + if (worker.pathContentsGood(i)) continue; printMsg(lvlError, format("found corrupted or missing path ‘%1%’ in the output closure of ‘%2%’") % i % drvPath); Path drvPath2 = outputsToDrv[i]; if (drvPath2 == "") @@ -1313,12 +1323,6 @@ void DerivationGoal::tryToBuild() deletePath(path); } - /* Check again whether any output previously failed to build, - because some other process may have tried and failed before we - acquired the lock. */ - for (auto & i : drv->outputs) - if (pathFailed(i.second.path)) return; - /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ @@ -1363,9 +1367,6 @@ void DerivationGoal::tryToBuild() printMsg(lvlError, e.msg()); outputLocks.unlock(); buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 0 % e.msg()); worker.permanentFailure = true; done(BuildResult::InputRejected, e.msg()); return; @@ -1403,22 +1404,14 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe --- just don't do that :-) */ - int status; - pid_t savedPid; - if (hook) { - savedPid = hook->pid; - status = hook->pid.wait(true); - } else { - /* !!! this could block! security problem! solution: kill the - child */ - savedPid = pid; - status = pid.wait(true); - } + /* !!! this could block! security problem! solution: kill the + child */ + int status = hook ? hook->pid.wait(true) : pid.wait(true); debug(format("builder process for ‘%1%’ finished") % drvPath); /* So the child is gone now. */ - worker.childTerminated(savedPid); + worker.childTerminated(shared_from_this()); /* Close the read side of the logger pipe. */ if (hook) { @@ -1469,11 +1462,19 @@ void DerivationGoal::buildDone() if (pathExists(chrootRootDir + i)) rename((chrootRootDir + i).c_str(), i.c_str()); + std::string msg = (format("builder for ‘%1%’ %2%") + % drvPath % statusToString(status)).str(); + + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto & line : logTail) + msg += "\n " + line; + } + if (diskFull) - printMsg(lvlError, "note: build failure may have been caused by lack of free disk space"); + msg += "\nnote: build failure may have been caused by lack of free disk space"; - throw BuildError(format("builder for ‘%1%’ %2%") - % drvPath % statusToString(status)); + throw BuildError(msg); } /* Compute the FS closure of the outputs and register them as @@ -1518,39 +1519,18 @@ void DerivationGoal::buildDone() BuildResult::Status st = BuildResult::MiscFailure; - if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - timeout") % drvPath); + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) st = BuildResult::TimedOut; - } else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ hook-failed %1% - %2% %3%") - % drvPath % status % e.msg()); } else { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - %2% %3%") - % drvPath % 1 % e.msg()); - st = dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : fixedOutput || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; - - /* Register the outputs of this build as "failed" so we - won't try to build them again (negative caching). - However, don't do this for fixed-output derivations, - since they're likely to fail for transient reasons - (e.g., fetchurl not being able to access the network). - Hook errors (like communication problems with the - remote machine) shouldn't be cached either. */ - if (settings.cacheFailure && !fixedOutput && !diskFull) - for (auto & i : drv->outputs) - worker.store.registerFailedPath(i.second.path); } done(st, e.msg()); @@ -1560,9 +1540,6 @@ void DerivationGoal::buildDone() /* Release the build user, if applicable. */ buildUser.release(); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath); - done(BuildResult::Built); } @@ -1637,11 +1614,7 @@ HookReply DerivationGoal::tryBuildHook() set<int> fds; fds.insert(hook->fromHook.readSide); fds.insert(hook->builderOut.readSide); - worker.childStarted(shared_from_this(), hook->pid, fds, false, false); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv->platform % logFile); + worker.childStarted(shared_from_this(), fds, false, false); return rpAccept; } @@ -1669,12 +1642,10 @@ void DerivationGoal::startBuilder() nrRounds > 1 ? "building path(s) %1% (round %2%/%3%)" : "building path(s) %1%"); f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); - startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); + printMsg(lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds); /* Right platform? */ if (!drv->canBuildLocally()) { - if (settings.printBuildTrace) - printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv->platform); throw Error( format("a ‘%1%’ is required to build ‘%3%’, but I am a ‘%2%’") % drv->platform % settings.thisSystem % drvPath); @@ -2177,7 +2148,7 @@ void DerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); builderOut.writeSide.close(); - worker.childStarted(shared_from_this(), pid, + worker.childStarted(shared_from_this(), singleton<set<int> >(builderOut.readSide), true, true); /* Check if setting up the build environment failed. */ @@ -2189,11 +2160,6 @@ void DerivationGoal::startBuilder() } printMsg(lvlDebug, msg); } - - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ build-started %1% - %2% %3%") - % drvPath % drv->platform % logFile); - } } @@ -2204,8 +2170,6 @@ void DerivationGoal::runChild() try { /* child */ - logType = ltFlat; - commonChildInit(builderOut); #if __linux__ @@ -2386,6 +2350,12 @@ void DerivationGoal::runChild() if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); #endif + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + /* Fill in the environment. */ Strings envStrs; for (auto & i : env) @@ -2541,7 +2511,6 @@ void DerivationGoal::runChild() /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { - logType = ltFlat; if (drv->builder == "builtin:fetchurl") builtinFetchurl(*drv); else @@ -2673,8 +2642,7 @@ void DerivationGoal::registerOutputs() rewritten = true; } - startNest(nest, lvlTalkative, - format("scanning for references inside ‘%1%’") % path); + Activity act(*logger, lvlTalkative, format("scanning for references inside ‘%1%’") % path); /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified @@ -2730,7 +2698,7 @@ void DerivationGoal::registerOutputs() if (buildMode == bmCheck) { if (!worker.store.isValidPath(path)) continue; - ValidPathInfo info = worker.store.queryPathInfo(path); + auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { if (settings.keepFailed) { Path dst = path + checkSuffix; @@ -2743,6 +2711,15 @@ void DerivationGoal::registerOutputs() throw Error(format("derivation ‘%1%’ may not be deterministic: output ‘%2%’ differs") % drvPath % path); } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + continue; } @@ -2772,14 +2749,25 @@ void DerivationGoal::registerOutputs() } else used = references; + PathSet badPaths; + for (auto & i : used) if (allowed) { if (spec.find(i) == spec.end()) - throw BuildError(format("output ‘%1%’ is not allowed to refer to path ‘%2%’") % actualPath % i); + badPaths.insert(i); } else { if (spec.find(i) != spec.end()) - throw BuildError(format("output ‘%1%’ is not allowed to refer to path ‘%2%’") % actualPath % i); + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n\t"; + badPathsStr += i; } + throw BuildError(format("output ‘%1%’ is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); + } }; checkRefs("allowedReferences", true, false); @@ -2790,7 +2778,7 @@ void DerivationGoal::registerOutputs() if (curRound == nrRounds) { worker.store.optimisePath(path); // FIXME: combine with scanForReferences() - worker.store.markContentsGood(path); + worker.markContentsGood(path); } ValidPathInfo info; @@ -2799,6 +2787,9 @@ void DerivationGoal::registerOutputs() info.narSize = hash.second; info.references = references; info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + infos.push_back(info); } @@ -2937,8 +2928,18 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) done(BuildResult::LogLimitExceeded); return; } - if (verbosity >= settings.buildVerbosity) - writeToStderr(filterANSIEscapes(data, true)); + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + if (bzLogFile) { int err; BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size()); @@ -2948,16 +2949,30 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) } if (hook && fd == hook->fromHook.readSide) - writeToStderr(data); + printMsg(lvlError, data); // FIXME? } void DerivationGoal::handleEOF(int fd) { + if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); } +void DerivationGoal::flushLine() +{ + if (settings.verboseBuild) + printMsg(lvlInfo, filterANSIEscapes(currentLogLine, true)); + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); + } + currentLogLine = ""; + currentLogLinePos = 0; +} + + PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { PathSet result; @@ -2965,30 +2980,13 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) if (!wantOutput(i.first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i.second.path) && - (!checkHash || worker.store.pathContentsGood(i.second.path)); + (!checkHash || worker.pathContentsGood(i.second.path)); if (good == returnValid) result.insert(i.second.path); } return result; } -bool DerivationGoal::pathFailed(const Path & path) -{ - if (!settings.cacheFailure) return false; - - if (!worker.store.hasPathFailed(path)) return false; - - printMsg(lvlError, format("builder for ‘%1%’ failed previously (cached)") % path); - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ build-failed %1% - cached") % drvPath); - - done(BuildResult::CachedFailure); - - return true; -} - - Path DerivationGoal::addHashRewrite(const Path & path) { string h1 = string(path, settings.nixStore.size() + 1, 32); @@ -3010,7 +3008,7 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg) amDone(result.success() ? ecSuccess : ecFailed); if (result.status == BuildResult::TimedOut) worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure || result.status == BuildResult::CachedFailure) + if (result.status == BuildResult::PermanentFailure) worker.permanentFailure = true; } @@ -3027,28 +3025,24 @@ private: Path storePath; /* The remaining substituters. */ - Paths subs; + std::list<ref<Store>> subs; /* The current substituter. */ - Path sub; + std::shared_ptr<Store> sub; - /* Whether any substituter can realise this path */ + /* Whether any substituter can realise this path. */ bool hasSubstitute; /* Path info returned by the substituter's query info operation. */ - SubstitutablePathInfo info; + std::shared_ptr<const ValidPathInfo> info; /* Pipe for the substituter's standard output. */ Pipe outPipe; - /* Pipe for the substituter's standard error. */ - Pipe logPipe; - - /* The process ID of the builder. */ - Pid pid; + /* The substituter thread. */ + std::thread thr; - /* Lock on the store path. */ - std::shared_ptr<PathLocks> outputLock; + std::promise<void> promise; /* Whether to try to repair a valid path. */ bool repair; @@ -3064,7 +3058,7 @@ public: SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false); ~SubstitutionGoal(); - void timedOut(); + void timedOut() { abort(); }; string key() { @@ -3105,20 +3099,14 @@ SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool SubstitutionGoal::~SubstitutionGoal() { - if (pid != -1) worker.childTerminated(pid); -} - - -void SubstitutionGoal::timedOut() -{ - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath); - if (pid != -1) { - pid_t savedPid = pid; - pid.kill(); - worker.childTerminated(savedPid); + try { + if (thr.joinable()) { + thr.join(); + worker.childTerminated(shared_from_this()); + } + } catch (...) { + ignoreException(); } - amDone(ecFailed); } @@ -3143,7 +3131,7 @@ void SubstitutionGoal::init() if (settings.readOnlyMode) throw Error(format("cannot substitute path ‘%1%’ - no write access to the Nix store") % storePath); - subs = settings.substituters; + subs = getDefaultSubstituters(); tryNext(); } @@ -3168,17 +3156,19 @@ void SubstitutionGoal::tryNext() sub = subs.front(); subs.pop_front(); - SubstitutablePathInfos infos; - PathSet dummy(singleton<PathSet>(storePath)); - worker.store.querySubstitutablePathInfos(sub, dummy, infos); - SubstitutablePathInfos::iterator k = infos.find(storePath); - if (k == infos.end()) { tryNext(); return; } - info = k->second; + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath &) { + tryNext(); + return; + } + hasSubstitute = true; /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - for (auto & i : info.references) + for (auto & i : info->references) if (i != storePath) /* ignore self-references */ addWaitee(worker.makeSubstitutionGoal(i)); @@ -3199,7 +3189,7 @@ void SubstitutionGoal::referencesValid() return; } - for (auto & i : info.references) + for (auto & i : info->references) if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); @@ -3221,75 +3211,32 @@ void SubstitutionGoal::tryToRun() return; } - /* Maybe a derivation goal has already locked this path - (exceedingly unlikely, since it should have used a substitute - first, but let's be defensive). */ - outputLock.reset(); // make sure this goal's lock is gone - if (pathIsLockedByMe(storePath)) { - debug(format("restarting substitution of ‘%1%’ because it's locked by another goal") - % storePath); - worker.waitForAnyGoal(shared_from_this()); - return; /* restart in the tryToRun() state when another goal finishes */ - } - - /* Acquire a lock on the output path. */ - outputLock = std::make_shared<PathLocks>(); - if (!outputLock->lockPaths(singleton<PathSet>(storePath), "", false)) { - worker.waitForAWhile(shared_from_this()); - return; - } - - /* Check again whether the path is invalid. */ - if (!repair && worker.store.isValidPath(storePath)) { - debug(format("store path ‘%1%’ has become valid") % storePath); - outputLock->setDeletion(true); - amDone(ecSuccess); - return; - } - printMsg(lvlInfo, format("fetching path ‘%1%’...") % storePath); outPipe.create(); - logPipe.create(); - - destPath = repair ? storePath + ".tmp" : storePath; - - /* Remove the (stale) output path if it exists. */ - deletePath(destPath); - - worker.store.setSubstituterEnv(); - /* Fill in the arguments. */ - Strings args; - args.push_back(baseNameOf(sub)); - args.push_back("--substitute"); - args.push_back(storePath); - args.push_back(destPath); + promise = std::promise<void>(); - /* Fork the substitute program. */ - pid = startProcess([&]() { - - commonChildInit(logPipe); + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide.close(); }); - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); + StringSink sink; + sub->exportPaths({storePath}, false, sink); - execv(sub.c_str(), stringsToCharPtrs(args).data()); + StringSource source(*sink.s); + worker.store.importPaths(false, source, 0); - throw SysError(format("executing ‘%1%’") % sub); + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } }); - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - outPipe.writeSide.close(); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton<set<int> >(logPipe.readSide), true, true); + worker.childStarted(shared_from_this(), {outPipe.readSide}, true, false); state = &SubstitutionGoal::finished; - - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub); } @@ -3297,110 +3244,40 @@ void SubstitutionGoal::finished() { trace("substitute finished"); - /* Since we got an EOF on the logger pipe, the substitute is - presumed to have terminated. */ - pid_t savedPid = pid; - int status = pid.wait(true); - - /* So the child is gone now. */ - worker.childTerminated(savedPid); - - /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); + thr.join(); + worker.childTerminated(shared_from_this()); - /* Get the hash info from stdout. */ - string dummy = readLine(outPipe.readSide); - string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : ""; - outPipe.readSide.close(); - - /* Check the exit status and the build result. */ - HashResult hash; try { - - if (!statusOk(status)) - throw SubstError(format("fetching path ‘%1%’ %2%") - % storePath % statusToString(status)); - - if (!pathExists(destPath)) - throw SubstError(format("substitute did not produce path ‘%1%’") % destPath); - - hash = hashPath(htSHA256, destPath); - - /* Verify the expected hash we got from the substituer. */ - if (expectedHashStr != "") { - size_t n = expectedHashStr.find(':'); - if (n == string::npos) - throw Error(format("bad hash from substituter: %1%") % expectedHashStr); - HashType hashType = parseHashType(string(expectedHashStr, 0, n)); - if (hashType == htUnknown) - throw Error(format("unknown hash algorithm in ‘%1%’") % expectedHashStr); - Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1)); - Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first; - if (expectedHash != actualHash) - throw SubstError(format("hash mismatch in downloaded path ‘%1%’: expected %2%, got %3%") - % storePath % printHash(expectedHash) % printHash(actualHash)); - } - - } catch (SubstError & e) { - + promise.get_future().get(); + } catch (Error & e) { printMsg(lvlInfo, e.msg()); - if (settings.printBuildTrace) { - printMsg(lvlError, format("@ substituter-failed %1% %2% %3%") - % storePath % status % e.msg()); - } - /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; worker.wakeUp(shared_from_this()); return; } - if (repair) replaceValidPath(storePath, destPath); - - canonicalisePathMetaData(storePath, -1); - - worker.store.optimisePath(storePath); // FIXME: combine with hashPath() - - ValidPathInfo info2; - info2.path = storePath; - info2.narHash = hash.first; - info2.narSize = hash.second; - info2.references = info.references; - info2.deriver = info.deriver; - worker.store.registerValidPath(info2); - - outputLock->setDeletion(true); - outputLock.reset(); - - worker.store.markContentsGood(storePath); + worker.markContentsGood(storePath); printMsg(lvlChatty, format("substitution of path ‘%1%’ succeeded") % storePath); - if (settings.printBuildTrace) - printMsg(lvlError, format("@ substituter-succeeded %1%") % storePath); - amDone(ecSuccess); } void SubstitutionGoal::handleChildOutput(int fd, const string & data) { - assert(fd == logPipe.readSide); - if (verbosity >= settings.buildVerbosity) writeToStderr(data); - /* Don't write substitution output to a log file for now. We - probably should, though. */ } void SubstitutionGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + if (fd == outPipe.readSide) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -3516,9 +3393,8 @@ unsigned Worker::getNrLocalBuilds() } -void Worker::childStarted(GoalPtr goal, - pid_t pid, const set<int> & fds, bool inBuildSlot, - bool respectTimeouts) +void Worker::childStarted(GoalPtr goal, const set<int> & fds, + bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; @@ -3526,30 +3402,29 @@ void Worker::childStarted(GoalPtr goal, child.timeStarted = child.lastOutput = time(0); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; - children[pid] = child; + children.emplace_back(child); if (inBuildSlot) nrLocalBuilds++; } -void Worker::childTerminated(pid_t pid, bool wakeSleepers) +void Worker::childTerminated(GoalPtr goal, bool wakeSleepers) { - assert(pid != -1); /* common mistake */ - - Children::iterator i = children.find(pid); + auto i = std::find_if(children.begin(), children.end(), + [&](const Child & child) { return child.goal.lock() == goal; }); assert(i != children.end()); - if (i->second.inBuildSlot) { + if (i->inBuildSlot) { assert(nrLocalBuilds > 0); nrLocalBuilds--; } - children.erase(pid); + children.erase(i); if (wakeSleepers) { /* Wake up goals waiting for a build slot. */ - for (auto & i : wantingToBuild) { - GoalPtr goal = i.lock(); + for (auto & j : wantingToBuild) { + GoalPtr goal = j.lock(); if (goal) wakeUp(goal); } @@ -3586,7 +3461,7 @@ void Worker::run(const Goals & _topGoals) { for (auto & i : _topGoals) topGoals.insert(i); - startNest(nest, lvlDebug, format("entered goal loop")); + Activity act(*logger, lvlDebug, "entered goal loop"); while (1) { @@ -3651,11 +3526,11 @@ void Worker::waitForInput() assert(sizeof(time_t) >= sizeof(long)); time_t nearest = LONG_MAX; // nearest deadline for (auto & i : children) { - if (!i.second.respectTimeouts) continue; + if (!i.respectTimeouts) continue; if (settings.maxSilentTime != 0) - nearest = std::min(nearest, i.second.lastOutput + settings.maxSilentTime); + nearest = std::min(nearest, i.lastOutput + settings.maxSilentTime); if (settings.buildTimeout != 0) - nearest = std::min(nearest, i.second.timeStarted + settings.buildTimeout); + nearest = std::min(nearest, i.timeStarted + settings.buildTimeout); } if (nearest != LONG_MAX) { timeout.tv_sec = std::max((time_t) 1, nearest - before); @@ -3673,7 +3548,6 @@ void Worker::waitForInput() timeout.tv_sec = std::max((time_t) 1, (time_t) (lastWokenUp + settings.pollInterval - before)); } else lastWokenUp = 0; - using namespace std; /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ @@ -3681,7 +3555,7 @@ void Worker::waitForInput() FD_ZERO(&fds); int fdMax = 0; for (auto & i : children) { - for (auto & j : i.second.fds) { + for (auto & j : i.fds) { FD_SET(j, &fds); if (j >= fdMax) fdMax = j + 1; } @@ -3695,22 +3569,16 @@ void Worker::waitForInput() time_t after = time(0); /* Process all available file descriptors. */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); - /* Since goals may be canceled from inside the loop below (causing - them go be erased from the `children' map), we have to be - careful that we don't keep iterators alive across calls to - timedOut(). */ - set<pid_t> pids; - for (auto & i : children) pids.insert(i.first); - - for (auto & i : pids) { checkInterrupt(); - Children::iterator j = children.find(i); - if (j == children.end()) continue; // child destroyed - GoalPtr goal = j->second.goal.lock(); + + GoalPtr goal = j->goal.lock(); assert(goal); - set<int> fds2(j->second.fds); + set<int> fds2(j->fds); for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { unsigned char buffer[4096]; @@ -3722,12 +3590,12 @@ void Worker::waitForInput() } else if (rd == 0) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); - j->second.fds.erase(k); + j->fds.erase(k); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); string data((char *) buffer, rd); - j->second.lastOutput = after; + j->lastOutput = after; goal->handleChildOutput(k, data); } } @@ -3735,8 +3603,8 @@ void Worker::waitForInput() if (goal->getExitCode() == Goal::ecBusy && settings.maxSilentTime != 0 && - j->second.respectTimeouts && - after - j->second.lastOutput >= (time_t) settings.maxSilentTime) + j->respectTimeouts && + after - j->lastOutput >= (time_t) settings.maxSilentTime) { printMsg(lvlError, format("%1% timed out after %2% seconds of silence") @@ -3746,8 +3614,8 @@ void Worker::waitForInput() else if (goal->getExitCode() == Goal::ecBusy && settings.buildTimeout != 0 && - j->second.respectTimeouts && - after - j->second.timeStarted >= (time_t) settings.buildTimeout) + j->respectTimeouts && + after - j->timeStarted >= (time_t) settings.buildTimeout) { printMsg(lvlError, format("%1% timed out after %2% seconds") @@ -3773,13 +3641,37 @@ unsigned int Worker::exitStatus() } +bool Worker::pathContentsGood(const Path & path) +{ + std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path ‘%1%’...") % path); + auto info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info->narHash.type, path); + Hash nullHash(htSHA256); + res = info->narHash == nullHash || info->narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); + return res; +} + + +void Worker::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + ////////////////////////////////////////////////////////////////////// void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - startNest(nest, lvlDebug, format("building %1%") % showPaths(drvPaths)); - Worker worker(*this); Goals goals; @@ -3809,8 +3701,6 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - startNest(nest, lvlDebug, format("building %1%") % showPaths({drvPath})); - Worker worker(*this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); @@ -3855,7 +3745,7 @@ void LocalStore::repairPath(const Path & path) if (goal->getExitCode() != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ - Path deriver = queryDeriver(path); + auto deriver = queryPathInfo(path)->deriver; if (deriver != "" && isValidPath(deriver)) { goals.clear(); goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); diff --git a/src/libstore/builtins.cc b/src/libstore/builtins.cc index c22c44f3c..a4785d690 100644 --- a/src/libstore/builtins.cc +++ b/src/libstore/builtins.cc @@ -20,6 +20,7 @@ void builtinFetchurl(const BasicDerivation & drv) options.showProgress = DownloadOptions::yes; auto data = makeDownloader()->download(url->second, options); + assert(data.data); auto out = drv.env.find("out"); if (out == drv.env.end()) throw Error("attribute ‘url’ missing"); @@ -29,12 +30,12 @@ void builtinFetchurl(const BasicDerivation & drv) auto unpack = drv.env.find("unpack"); if (unpack != drv.env.end() && unpack->second == "1") { - if (string(data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) - data.data = decompressXZ(data.data); - StringSource source(data.data); + if (string(*data.data, 0, 6) == string("\xfd" "7zXZ\0", 6)) + data.data = decompress("xz", ref<std::string>(data.data)); + StringSource source(*data.data); restorePath(storePath, source); } else - writeFile(storePath, data.data); + writeFile(storePath, *data.data); auto executable = drv.env.find("executable"); if (executable != drv.env.end() && executable->second == "1") { diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index caba22c1e..747483afb 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,5 +1,6 @@ #include "crypto.hh" #include "util.hh" +#include "globals.hh" #if HAVE_SODIUM #include <sodium.h> @@ -98,4 +99,28 @@ bool verifyDetached(const std::string & data, const std::string & sig, #endif } +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.get("binary-cache-public-keys", Strings())) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.get("secret-key-files", Strings())) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + } diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 38d5fe2a8..9110af3aa 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -49,4 +49,6 @@ typedef std::map<std::string, PublicKey> PublicKeys; bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); +PublicKeys getDefaultPublicKeys(); + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d9b009d40..becf85245 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -290,7 +290,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv) DerivationInputs inputs2; for (auto & i : drv.inputDrvs) { Hash h = drvHashes[i.first]; - if (h.type == htUnknown) { + if (!h) { assert(store.isValidPath(i.first)); Derivation drv2 = readDerivation(i.first); h = hashDerivationModulo(store, drv2); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 4776d0091..6e39330e4 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -18,10 +18,18 @@ double getTime() return tv.tv_sec + (tv.tv_usec / 1000000.0); } +std::string resolveUri(const std::string & uri) +{ + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + else + return uri; +} + struct CurlDownloader : public Downloader { CURL * curl; - string data; + ref<std::string> data; string etag, status, expectedETag; struct curl_slist * requestHeaders; @@ -33,7 +41,7 @@ struct CurlDownloader : public Downloader size_t writeCallback(void * contents, size_t size, size_t nmemb) { size_t realSize = size * nmemb; - data.append((char *) contents, realSize); + data->append((char *) contents, realSize); return realSize; } @@ -102,6 +110,7 @@ struct CurlDownloader : public Downloader } CurlDownloader() + : data(make_ref<std::string>()) { requestHeaders = 0; @@ -148,7 +157,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); } - data.clear(); + data->clear(); if (requestHeaders) { curl_slist_free_all(requestHeaders); @@ -179,11 +188,9 @@ struct CurlDownloader : public Downloader if (res == CURLE_WRITE_ERROR && etag == options.expectedETag) return false; long httpStatus = -1; - if (res == CURLE_HTTP_RETURNED_ERROR) - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &httpStatus); if (res != CURLE_OK) { - long httpStatus = 0; Error err = httpStatus == 404 ? NotFound : httpStatus == 403 ? Forbidden : Misc; @@ -199,7 +206,7 @@ struct CurlDownloader : public Downloader DownloadResult download(string url, const DownloadOptions & options) override { DownloadResult res; - if (fetch(url, options)) { + if (fetch(resolveUri(url), options)) { res.cached = false; res.data = data; } else @@ -209,16 +216,16 @@ struct CurlDownloader : public Downloader } }; - ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); } - -Path Downloader::downloadCached(ref<Store> store, const string & url, bool unpack) +Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack) { - Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs"; + auto url = resolveUri(url_); + + Path cacheDir = getCacheDir() + "/nix/tarballs"; createDirs(cacheDir); string urlHash = printHash32(hashString(htSHA256, url)); @@ -263,7 +270,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url, bool unpac auto res = download(url, options); if (!res.cached) - storePath = store->addTextToStore(name, res.data, PathSet(), false); + storePath = store->addTextToStore(name, *res.data, PathSet(), false); assert(!storePath.empty()); replaceSymlink(storePath, fileLink); @@ -302,10 +309,11 @@ Path Downloader::downloadCached(ref<Store> store, const string & url, bool unpac bool isUri(const string & s) { + if (s.compare(0, 8, "channel:") == 0) return true; size_t pos = s.find("://"); if (pos == string::npos) return false; string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file"; + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git"; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 5dd2d2c82..eb2b76678 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -17,7 +17,8 @@ struct DownloadOptions struct DownloadResult { bool cached; - string data, etag; + string etag; + std::shared_ptr<std::string> data; }; class Store; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index e082f6714..8fc582f4c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -147,35 +147,36 @@ Path Store::addPermRoot(const Path & _storePath, void LocalStore::addTempRoot(const Path & path) { + auto state(_state.lock()); + /* Create the temporary roots file for this process. */ - if (fdTempRoots == -1) { + if (state->fdTempRoots == -1) { while (1) { Path dir = (format("%1%/%2%") % settings.nixStateDir % tempRootsDir).str(); createDirs(dir); - fnTempRoots = (format("%1%/%2%") - % dir % getpid()).str(); + state->fnTempRoots = (format("%1%/%2%") % dir % getpid()).str(); AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) + if (pathExists(state->fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ - unlink(fnTempRoots.c_str()); + unlink(state->fnTempRoots.c_str()); - fdTempRoots = openLockFile(fnTempRoots, true); + state->fdTempRoots = openLockFile(state->fnTempRoots, true); fdGCLock.close(); - debug(format("acquiring read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("acquiring read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); /* Check whether the garbage collector didn't get in our way. */ struct stat st; - if (fstat(fdTempRoots, &st) == -1) - throw SysError(format("statting ‘%1%’") % fnTempRoots); + if (fstat(state->fdTempRoots, &st) == -1) + throw SysError(format("statting ‘%1%’") % state->fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -187,15 +188,15 @@ void LocalStore::addTempRoot(const Path & path) /* Upgrade the lock to a write lock. This will cause us to block if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltWrite, true); + debug(format("acquiring write lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltWrite, true); string s = path + '\0'; - writeFull(fdTempRoots, s); + writeFull(state->fdTempRoots, s); /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on ‘%1%’") % fnTempRoots); - lockFile(fdTempRoots, ltRead, true); + debug(format("downgrading to read lock on ‘%1%’") % state->fnTempRoots); + lockFile(state->fdTempRoots, ltRead, true); } @@ -304,7 +305,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = settings.nixStore + "/" + baseNameOf(path); - if (isValidPath(storePath)) + if (isStorePath(storePath) && isValidPath(storePath)) roots[path] = storePath; } @@ -406,7 +407,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) deletePathRecursive(state, i); - size = queryPathInfo(path).narSize; + size = queryPathInfo(path)->narSize; invalidatePathChecked(path); } @@ -484,7 +485,7 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p if (state.gcKeepDerivations && isDerivation(path)) { PathSet outputs = queryDerivationOutputs(path); for (auto & i : outputs) - if (isValidPath(i) && queryDeriver(i) == path) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) incoming.insert(i); } @@ -513,7 +514,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) if (path == linksDir || path == state.trashDir) return; - startNest(nest, lvlDebug, format("considering whether to delete ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("considering whether to delete ‘%1%’") % path); if (!isValidPath(path)) { /* A lock file belonging to a path that we're building right @@ -690,7 +691,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) string name = dirent->d_name; if (name == "." || name == "..") continue; Path path = settings.nixStore + "/" + name; - if (isValidPath(path)) + if (isStorePath(path) && isValidPath(path)) entries.push_back(path); else tryToDelete(state, path); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e704837e8..c12178e40 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -28,7 +28,6 @@ Settings::Settings() keepFailed = false; keepGoing = false; tryFallback = false; - buildVerbosity = lvlError; maxBuildJobs = 1; buildCores = 1; #ifdef _SC_NPROCESSORS_ONLN @@ -40,7 +39,6 @@ Settings::Settings() maxSilentTime = 0; buildTimeout = 0; useBuildHook = true; - printBuildTrace = false; reservedSize = 8 * 1024 * 1024; fsyncMetadata = true; useSQLiteWAL = true; @@ -52,7 +50,6 @@ Settings::Settings() keepLog = true; compressLog = true; maxLogSize = 0; - cacheFailure = false; pollInterval = 5; checkRootReachability = false; gcKeepOutputs = false; @@ -175,7 +172,6 @@ void Settings::update() _get(keepLog, "build-keep-log"); _get(compressLog, "build-compress-log"); _get(maxLogSize, "build-max-log-size"); - _get(cacheFailure, "build-cache-failure"); _get(pollInterval, "build-poll-interval"); _get(checkRootReachability, "gc-check-reachability"); _get(gcKeepOutputs, "gc-keep-outputs"); @@ -188,20 +184,6 @@ void Settings::update() _get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); _get(useCaseHack, "use-case-hack"); _get(preBuildHook, "pre-build-hook"); - - string subs = getEnv("NIX_SUBSTITUTERS", "default"); - if (subs == "default") { - substituters.clear(); -#if 0 - if (getEnv("NIX_OTHER_STORES") != "") - substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); -#endif - substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); - substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); - if (useSshSubstituter && !sshSubstituterHosts.empty()) - substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); - } else - substituters = tokenizeString<Strings>(subs, ":"); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 60b11afe6..65f763ace 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <map> #include <sys/types.h> @@ -77,8 +78,12 @@ struct Settings { instead. */ bool tryFallback; - /* Verbosity level for build output. */ - Verbosity buildVerbosity; + /* Whether to show build log output in real time. */ + bool verboseBuild = true; + + /* If verboseBuild is false, the number of lines of the tail of + the log to show if a build fails. */ + size_t logLines = 10; /* Maximum number of parallel build jobs. 0 means unlimited. */ unsigned int maxBuildJobs; @@ -105,31 +110,10 @@ struct Settings { means infinity. */ time_t buildTimeout; - /* The substituters. There are programs that can somehow realise - a store path without building, e.g., by downloading it or - copying it from a CD. */ - Paths substituters; - /* Whether to use build hooks (for distributed builds). Sometimes users want to disable this from the command-line. */ bool useBuildHook; - /* Whether buildDerivations() should print out lines on stderr in - a fixed format to allow its progress to be monitored. Each - line starts with a "@". The following are defined: - - @ build-started <drvpath> <outpath> <system> <logfile> - @ build-failed <drvpath> <outpath> <exitcode> <error text> - @ build-succeeded <drvpath> <outpath> - @ substituter-started <outpath> <substituter> - @ substituter-failed <outpath> <exitcode> <error text> - @ substituter-succeeded <outpath> - - Best combined with --no-build-output, otherwise stderr might - conceivably contain lines in this format printed by the - builders. */ - bool printBuildTrace; - /* Amount of reserved space for the garbage collector (/nix/var/nix/db/reserved). */ off_t reservedSize; @@ -168,9 +152,6 @@ struct Settings { before being killed (0 means no limit). */ unsigned long maxLogSize; - /* Whether to cache build failures. */ - bool cacheFailure; - /* How often (in seconds) to poll for locks. */ unsigned int pollInterval; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1d707c043..92d94aeea 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,6 +1,7 @@ #include "binary-cache-store.hh" #include "download.hh" #include "globals.hh" +#include "nar-info-disk-cache.hh" namespace nix { @@ -10,25 +11,37 @@ private: Path cacheUri; - ref<Downloader> downloader; + Pool<Downloader> downloaders; public: HttpBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & _cacheUri) - : BinaryCacheStore(localStore, secretKeyFile) + const StoreParams & params, const Path & _cacheUri) + : BinaryCacheStore(localStore, params) , cacheUri(_cacheUri) - , downloader(makeDownloader()) + , downloaders( + std::numeric_limits<size_t>::max(), + []() { return makeDownloader(); }) { if (cacheUri.back() == '/') cacheUri.pop_back(); + + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override + { + return cacheUri; } void init() override { // FIXME: do this lazily? - if (!fileExists("nix-cache-info")) - throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + if (!diskCache->cacheExists(cacheUri)) { + if (!fileExists("nix-cache-info")) + throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri); + diskCache->createCache(cacheUri); + } } protected: @@ -36,13 +49,16 @@ protected: bool fileExists(const std::string & path) override { try { + auto downloader(downloaders.get()); DownloadOptions options; options.showProgress = DownloadOptions::no; options.head = true; downloader->download(cacheUri + "/" + path, options); return true; } catch (DownloadError & e) { - if (e.error == Downloader::NotFound) + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) return false; throw; } @@ -53,21 +69,30 @@ protected: throw Error("uploading to an HTTP binary cache is not supported"); } - std::string getFile(const std::string & path) override + std::shared_ptr<std::string> getFile(const std::string & path) override { + auto downloader(downloaders.get()); DownloadOptions options; options.showProgress = DownloadOptions::no; - return downloader->download(cacheUri + "/" + path, options).data; + try { + return downloader->download(cacheUri + "/" + path, options).data; + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return 0; + throw; + } } }; -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ if (std::string(uri, 0, 7) != "http://" && std::string(uri, 0, 8) != "https://") return 0; auto store = std::make_shared<HttpBinaryCacheStore>(std::shared_ptr<Store>(0), - settings.get("binary-cache-secret-key-file", string("")), - uri); + params, uri); store->init(); return store; }); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index efd6d4725..2c2944938 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -12,26 +12,42 @@ private: public: LocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir); + const StoreParams & params, const Path & binaryCacheDir) + : BinaryCacheStore(localStore, params) + , binaryCacheDir(binaryCacheDir) + { + } void init() override; + std::string getUri() override + { + return "file://" + binaryCacheDir; + } + protected: bool fileExists(const std::string & path) override; void upsertFile(const std::string & path, const std::string & data) override; - std::string getFile(const std::string & path) override; + std::shared_ptr<std::string> getFile(const std::string & path) override; -}; + PathSet queryAllValidPaths() override + { + PathSet paths; -LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir) - : BinaryCacheStore(localStore, secretKeyFile) - , binaryCacheDir(binaryCacheDir) -{ -} + for (auto & entry : readDirectory(binaryCacheDir)) { + if (entry.name.size() != 40 || + !hasSuffix(entry.name, ".narinfo")) + continue; + paths.insert(settings.nixStore + "/" + entry.name.substr(0, entry.name.size() - 8)); + } + + return paths; + } + +}; void LocalBinaryCacheStore::init() { @@ -59,25 +75,25 @@ void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::stri atomicWrite(binaryCacheDir + "/" + path, data); } -std::string LocalBinaryCacheStore::getFile(const std::string & path) +std::shared_ptr<std::string> LocalBinaryCacheStore::getFile(const std::string & path) { - return readFile(binaryCacheDir + "/" + path); + try { + return std::make_shared<std::string>(readFile(binaryCacheDir + "/" + path)); + } catch (SysError & e) { + if (e.errNo == ENOENT) return 0; + throw; + } } -ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir) +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> { - auto store = make_ref<LocalBinaryCacheStore>( - localStore, secretKeyFile, binaryCacheDir); + if (std::string(uri, 0, 7) != "file://") return 0; + auto store = std::make_shared<LocalBinaryCacheStore>( + std::shared_ptr<Store>(0), params, std::string(uri, 7)); store->init(); return store; -} - -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { - if (std::string(uri, 0, 7) != "file://") return 0; - return openLocalBinaryCacheStore(std::shared_ptr<Store>(0), - settings.get("binary-cache-secret-key-file", string("")), - std::string(uri, 7)); }); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 7094a50a3..303c3af27 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,3 +1,4 @@ +#include "archive.hh" #include "fs-accessor.hh" #include "store-api.hh" @@ -22,7 +23,7 @@ struct LocalStoreAccessor : public FSAccessor struct stat st; if (lstat(path.c_str(), &st)) { - if (errno == ENOENT) return {Type::tMissing, 0, false}; + if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; throw SysError(format("getting status of ‘%1%’") % path); } @@ -68,4 +69,11 @@ ref<FSAccessor> LocalFSStore::getFSAccessor() return make_ref<LocalStoreAccessor>(ref<Store>(shared_from_this())); } +void LocalFSStore::narFromPath(const Path & path, Sink & sink) +{ + if (!isValidPath(path)) + throw Error(format("path ‘%s’ is not valid") % path); + dumpPath(path, sink); +} + } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8a2b7bb91..01a11f11f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,7 +5,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" -#include "affinity.hh" +#include "nar-info.hh" #include <iostream> #include <algorithm> @@ -36,168 +36,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -[[noreturn]] static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -217,20 +55,21 @@ void checkStoreNotSymlink() LocalStore::LocalStore() - : reservedPath(settings.nixDBPath + "/reserved") - , didSetSubstituterEnv(false) + : linksDir(settings.nixStore + "/.links") + , reservedPath(settings.nixDBPath + "/reserved") + , schemaPath(settings.nixDBPath + "/schema") { - schemaPath = settings.nixDBPath + "/schema"; + auto state(_state.lock()); if (settings.readOnlyMode) { - openDB(false); + openDB(*state, false); return; } /* Create missing state directories if they don't already exist. */ createDirs(settings.nixStore); makeStoreWritable(); - createDirs(linksDir = settings.nixStore + "/.links"); + createDirs(linksDir); Path profilesDir = settings.nixStateDir + "/profiles"; createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); @@ -302,7 +141,7 @@ LocalStore::LocalStore() } catch (SysError & e) { if (e.errNo != EACCES) throw; settings.readOnlyMode = true; - openDB(false); + openDB(*state, false); return; } @@ -320,7 +159,7 @@ LocalStore::LocalStore() else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - openDB(true); + openDB(*state, true); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } @@ -331,6 +170,12 @@ LocalStore::LocalStore() "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); + if (curSchema < 6) + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + if (!lockFile(globalLock, ltWrite, false)) { printMsg(lvlError, "waiting for exclusive access to the Nix store..."); lockFile(globalLock, ltWrite, true); @@ -340,37 +185,43 @@ LocalStore::LocalStore() have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 6) upgradeStore6(); - else if (curSchema < 7) { upgradeStore7(); openDB(true); } + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "upgrading database schema"); + txn.commit(); + } writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); } - else openDB(false); + else openDB(*state, false); } LocalStore::~LocalStore() { - try { - for (auto & i : runningSubstituters) { - if (i.second.disabled) continue; - i.second.to.close(); - i.second.from.close(); - i.second.error.close(); - if (i.second.pid != -1) - i.second.pid.wait(true); - } - } catch (...) { - ignoreException(); - } + auto state(_state.lock()); try { - if (fdTempRoots != -1) { - fdTempRoots.close(); - unlink(fnTempRoots.c_str()); + if (state->fdTempRoots != -1) { + state->fdTempRoots.close(); + unlink(state->fnTempRoots.c_str()); } } catch (...) { ignoreException(); @@ -378,6 +229,12 @@ LocalStore::~LocalStore() } +std::string LocalStore::getUri() +{ + return "local"; +} + + int LocalStore::getSchema() { int curSchema = 0; @@ -396,13 +253,14 @@ bool LocalStore::haveWriteAccess() } -void LocalStore::openDB(bool create) +void LocalStore::openDB(State & state, bool create) { if (!haveWriteAccess()) throw SysError(format("Nix database directory ‘%1%’ is not writable") % settings.nixDBPath); /* Open the Nix database. */ string dbPath = settings.nixDBPath + "/db.sqlite"; + auto & db(state.db); if (sqlite3_open_v2(dbPath.c_str(), &db.db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error(format("cannot open Nix database ‘%1%’") % dbPath); @@ -455,40 +313,31 @@ void LocalStore::openDB(bool create) } /* Prepare SQL statements. */ - stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); - stmtUpdatePathInfo.create(db, - "update ValidPaths set narSize = ?, hash = ? where path = ?;"); - stmtAddReference.create(db, + state.stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);"); + state.stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;"); + state.stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); - stmtQueryReferences.create(db, + state.stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;"); + state.stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - stmtQueryReferrers.create(db, + state.stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - stmtInvalidatePath.create(db, + state.stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); - stmtRegisterFailedPath.create(db, - "insert or ignore into FailedPaths (path, time) values (?, ?);"); - stmtHasPathFailed.create(db, - "select time from FailedPaths where path = ?;"); - stmtQueryFailedPaths.create(db, - "select path from FailedPaths;"); - // If the path is a derivation, then clear its outputs. - stmtClearFailedPath.create(db, - "delete from FailedPaths where ?1 = '*' or path = ?1 " - "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);"); - stmtAddDerivationOutput.create(db, + state.stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - stmtQueryValidDerivers.create(db, + state.stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - stmtQueryDerivationOutputs.create(db, + state.stmtQueryDerivationOutputs.create(db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - stmtQueryPathFromHashPart.create(db, + state.stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + state.stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -683,23 +532,19 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.narHash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path ‘%1%’ in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + state.stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.narHash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -716,89 +561,20 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che if (checkOutputs) checkDerivationOutputs(info.path, drv); for (auto & i : drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i.first); - stmtAddDerivationOutput.bind(i.second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for ‘%1%’ in database") % info.path); + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } - return id; -} - - -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) -{ - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); -} - - -void LocalStore::registerFailedPath(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path ‘%1%’") % path); - } end_retry_sqlite; -} - - -bool LocalStore::hasPathFailed(const Path & path) -{ - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; -} - - -PathSet LocalStore::queryFailedPaths() -{ - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); - - PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); - - return res; - } end_retry_sqlite; -} - - -void LocalStore::clearFailedPaths(const PathSet & paths) -{ - retry_sqlite { - SQLiteTxn txn(db); - - for (auto & i : paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path ‘%1%’ in database") % i); - } + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info)); + } - txn.commit(); - } end_retry_sqlite; + return id; } @@ -816,166 +592,124 @@ Hash parseHashField(const Path & path, const string & s) } -ValidPathInfo LocalStore::queryPathInfo(const Path & path) +std::shared_ptr<ValidPathInfo> LocalStore::queryPathInfoUncached(const Path & path) { - ValidPathInfo info; - info.path = path; + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; assertStorePath(path); - retry_sqlite { + return retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + auto state(_state.lock()); /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); - stmtQueryPathInfo.bind(path); + if (!useQueryPathInfo.next()) + return std::shared_ptr<ValidPathInfo>(); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + info->id = useQueryPathInfo.getInt(0); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info->narHash = parseHashField(path, useQueryPathInfo.getStr(1)); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.narHash = parseHashField(path, s); + info->registrationTime = useQueryPathInfo.getInt(2); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); - if (s) info.deriver = s; + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + if (s) info->deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info->narSize = useQueryPathInfo.getInt(4); - /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); + info->ultimate = useQueryPathInfo.getInt(5) == 1; - stmtQueryReferences.bind(info.id); + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info->sigs = tokenizeString<StringSet>(s, " "); - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferences.next()) + info->references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } -/* Update path info in the database. Currently only updates the - narSize field. */ -void LocalStore::updatePathInfo(const ValidPathInfo & info) +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.narHash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path ‘%1%’ in database") % info.path); + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.narHash)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path ‘%1%’ is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } -bool LocalStore::isValidPath_(const Path & path) +bool LocalStore::isValidPath_(State & state, const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return state.stmtQueryPathInfo.use()(path).next(); } -bool LocalStore::isValidPath(const Path & path) +bool LocalStore::isValidPathUncached(const Path & path) { - retry_sqlite { - return isValidPath_(path); - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { - PathSet res; - for (auto & i : paths) - if (isValidPath_(i)) res.insert(i); - return res; - } end_retry_sqlite; + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; + }); } -void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); - - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of ‘%1%’") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { - queryReferrers_(path, referrers); - } end_retry_sqlite; -} - - -Path LocalStore::queryDeriver(const Path & path) -{ - return queryPathInfo(path).deriver; + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); } @@ -983,67 +717,51 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of ‘%1%’") % path); + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); - PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of ‘%1%’") % path); + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); - StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of ‘%1%’") % path); + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } @@ -1053,224 +771,59 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); - - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); - - const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; -} - - -void LocalStore::setSubstituterEnv() -{ - if (didSetSubstituterEnv) return; + return retrySQLite<Path>([&]() { + auto state(_state.lock()); - /* Pass configuration options (including those overridden with - --option) to substituters. */ - setenv("_NIX_OPTIONS", settings.pack().c_str(), 1); - - didSetSubstituterEnv = true; -} - - -void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) -{ - if (run.disabled || run.pid != -1) return; + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - debug(format("starting substituter program ‘%1%’") % substituter); + if (!useQueryPathFromHashPart.next()) return ""; - Pipe toPipe, fromPipe, errorPipe; - - toPipe.create(); - fromPipe.create(); - errorPipe.create(); - - setSubstituterEnv(); - - run.pid = startProcess([&]() { - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing ‘%1%’") % substituter); + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; }); - - run.program = baseNameOf(substituter); - run.to = toPipe.writeSide.borrow(); - run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); - run.error = errorPipe.readSide.borrow(); - - toPipe.readSide.close(); - fromPipe.writeSide.close(); - errorPipe.writeSide.close(); - - /* The substituter may exit right away if it's disabled in any way - (e.g. copy-from-other-stores.pl will exit if no other stores - are configured). */ - try { - getLineFromSubstituter(run); - } catch (EndOfFile & e) { - run.to.close(); - run.from.close(); - run.error.close(); - run.disabled = true; - if (run.pid.wait(true) != 0) throw; - } -} - - -/* Read a line from the substituter's stdout, while also processing - its stderr. */ -string LocalStore::getLineFromSubstituter(RunningSubstituter & run) -{ - string res, err; - - /* We might have stdout data left over from the last time. */ - if (run.fromBuf.hasData()) goto haveData; - - while (1) { - checkInterrupt(); - - fd_set fds; - FD_ZERO(&fds); - FD_SET(run.from, &fds); - FD_SET(run.error, &fds); - - /* Wait for data to appear on the substituter's stdout or - stderr. */ - if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) { - if (errno == EINTR) continue; - throw SysError("waiting for input from the substituter"); - } - - /* Completely drain stderr before dealing with stdout. */ - if (FD_ISSET(run.error, &fds)) { - char buf[4096]; - ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf)); - if (n == -1) { - if (errno == EINTR) continue; - throw SysError("reading from substituter's stderr"); - } - if (n == 0) throw EndOfFile(format("substituter ‘%1%’ died unexpectedly") % run.program); - err.append(buf, n); - string::size_type p; - while ((p = err.find('\n')) != string::npos) { - printMsg(lvlError, run.program + ": " + string(err, 0, p)); - err = string(err, p + 1); - } - } - - /* Read from stdout until we get a newline or the buffer is empty. */ - else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) { - haveData: - do { - unsigned char c; - run.fromBuf(&c, 1); - if (c == '\n') { - if (!err.empty()) printMsg(lvlError, run.program + ": " + err); - return res; - } - res += c; - } while (run.fromBuf.hasData()); - } - } -} - - -template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run) -{ - string s = getLineFromSubstituter(run); - T res; - if (!string2Int(s, res)) throw Error("integer expected from stream"); - return res; } PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { PathSet res; - for (auto & i : settings.substituters) { - if (res.size() == paths.size()) break; - RunningSubstituter & run(runningSubstituters[i]); - startSubstituter(i, run); - if (run.disabled) continue; - string s = "have "; - for (auto & j : paths) - if (res.find(j) == res.end()) { s += j; s += " "; } - writeLine(run.to, s); - while (true) { - /* FIXME: we only read stderr when an error occurs, so - substituters should only write (short) messages to - stderr when they fail. I.e. they shouldn't write debug - output. */ - Path path = getLineFromSubstituter(run); - if (path == "") break; - res.insert(path); + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (res.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + if (sub->isValidPath(path)) + res.insert(path); } } return res; } -void LocalStore::querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos) -{ - RunningSubstituter & run(runningSubstituters[substituter]); - startSubstituter(substituter, run); - if (run.disabled) return; - - string s = "info "; - for (auto & i : paths) - if (infos.find(i) == infos.end()) { s += i; s += " "; } - writeLine(run.to, s); - - while (true) { - Path path = getLineFromSubstituter(run); - if (path == "") break; - if (paths.find(path) == paths.end()) - throw Error(format("got unexpected path ‘%1%’ from substituter") % path); - paths.erase(path); - SubstitutablePathInfo & info(infos[path]); - info.deriver = getLineFromSubstituter(run); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLineFromSubstituter<int>(run); - while (nrRefs--) { - Path p = getLineFromSubstituter(run); - assertStorePath(p); - info.references.insert(p); - } - info.downloadSize = getIntLineFromSubstituter<long long>(run); - info.narSize = getIntLineFromSubstituter<long long>(run); - } -} - - void LocalStore::querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) { - PathSet todo = paths; - for (auto & i : settings.substituters) { - if (todo.empty()) break; - querySubstitutablePathInfos(i, todo, infos); + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path)) continue; + debug(format("checking substituter ‘%s’ for path ‘%s’") + % sub->getUri() % path); + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath) { + } + } } } -Hash LocalStore::queryPathHash(const Path & path) -{ - return queryPathInfo(path).narHash; -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { ValidPathInfos infos; @@ -1281,28 +834,31 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - /* SQLite will fsync by default, but the new valid paths may not be fsync-ed. - * So some may want to fsync them before registering the validity, at the - * expense of some speed of the path registering operation. */ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { - SQLiteTxn txn(db); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); PathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); - if (isValidPath_(i.path)) - updatePathInfo(i); + if (isValidPath_(*state, i.path)) + updatePathInfo(*state, i); else - addValidPath(i, false); + addValidPath(*state, i, false); paths.insert(i.path); } for (auto & i : infos) { - unsigned long long referrer = queryValidPathId(i.path); + auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - addReference(referrer, queryValidPathId(j)); + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1323,27 +879,25 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSortPaths(paths); txn.commit(); - } end_retry_sqlite; + }); } /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(const Path & path) +void LocalStore::invalidatePath(State & state, const Path & path) { debug(format("invalidating path ‘%1%’") % path); - drvHashes.erase(path); - - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path ‘%1%’ in database") % path); + state.stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } } @@ -1392,6 +946,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, info.path = dstPath; info.narHash = hash.first; info.narSize = hash.second; + info.ultimate = true; registerValidPath(info); } @@ -1406,7 +961,6 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, bool repair) { Path srcPath(absPath(_srcPath)); - debug(format("adding ‘%1%’ to the store") % srcPath); /* Read the whole path into memory. This is not a very scalable method for very large paths, but `copyPath' is mainly used for @@ -1451,6 +1005,7 @@ Path LocalStore::addTextToStore(const string & name, const string & s, info.narHash = hash; info.narSize = sink.s->size(); info.references = references; + info.ultimate = true; registerValidPath(info); } @@ -1497,8 +1052,7 @@ void LocalStore::exportPath(const Path & path, bool sign, printMsg(lvlTalkative, format("exporting path ‘%1%’") % path); - if (!isValidPath(path)) - throw Error(format("path ‘%1%’ is not valid") % path); + auto info = queryPathInfo(path); HashAndWriteSink hashAndWriteSink(sink); @@ -1508,15 +1062,11 @@ void LocalStore::exportPath(const Path & path, bool sign, filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - Hash storedHash = queryPathHash(path); - if (hash != storedHash && storedHash != Hash(storedHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error(format("hash of path ‘%1%’ has changed from ‘%2%’ to ‘%3%’!") % path - % printHash(storedHash) % printHash(hash)); - - PathSet references; - queryReferences(path, references); + % printHash(info->narHash) % printHash(hash)); - hashAndWriteSink << exportMagic << path << references << queryDeriver(path); + hashAndWriteSink << exportMagic << path << info->references << info->deriver; if (sign) { Hash hash = hashAndWriteSink.currentHash(); @@ -1707,20 +1257,22 @@ void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteTxn txn(db); + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); - if (isValidPath_(path)) { - PathSet referrers; queryReferrers_(path, referrers); + if (isValidPath_(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) throw PathInUse(format("cannot delete path ‘%1%’ because it is in use by %2%") % path % showPaths(referrers)); - invalidatePath(path); + invalidatePath(*state, path); } txn.commit(); - } end_retry_sqlite; + }); } @@ -1756,36 +1308,39 @@ bool LocalStore::verifyStore(bool checkContents, bool repair) for (auto & i : validPaths) { try { - ValidPathInfo info = queryPathInfo(i); + auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of ‘%1%’") % i); - HashResult current = hashPath(info.narHash.type, i); + HashResult current = hashPath(info->narHash.type, i); - if (info.narHash != nullHash && info.narHash != current.first) { + if (info->narHash != nullHash && info->narHash != current.first) { printMsg(lvlError, format("path ‘%1%’ was modified! " "expected hash ‘%2%’, got ‘%3%’") - % i % printHash(info.narHash) % printHash(current.first)); + % i % printHash(info->narHash) % printHash(current.first)); if (repair) repairPath(i); else errors = true; } else { bool update = false; /* Fill in missing hashes. */ - if (info.narHash == nullHash) { + if (info->narHash == nullHash) { printMsg(lvlError, format("fixing missing hash on ‘%1%’") % i); - info.narHash = current.first; + info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ - if (info.narSize == 0) { + if (info->narSize == 0) { printMsg(lvlError, format("updating size field on ‘%1%’ to %2%") % i % current.second); - info.narSize = current.second; + info->narSize = current.second; update = true; } - if (update) updatePathInfo(info); + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } } @@ -1815,7 +1370,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (!isStorePath(path)) { printMsg(lvlError, format("path ‘%1%’ is not in the Nix store") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); return; } @@ -1833,7 +1389,8 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, if (canInvalidate) { printMsg(lvlError, format("path ‘%1%’ disappeared, removing from database...") % path); - invalidatePath(path); + auto state(_state.lock()); + invalidatePath(*state, path); } else { printMsg(lvlError, format("path ‘%1%’ disappeared, but it still has valid referrers!") % path); if (repair) @@ -1853,114 +1410,6 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } -bool LocalStore::pathContentsGood(const Path & path) -{ - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); - if (i != pathContentsGoodCache.end()) return i->second; - printMsg(lvlInfo, format("checking path ‘%1%’...") % path); - ValidPathInfo info = queryPathInfo(path); - bool res; - if (!pathExists(path)) - res = false; - else { - HashResult current = hashPath(info.narHash.type, path); - Hash nullHash(htSHA256); - res = info.narHash == nullHash || info.narHash == current.first; - } - pathContentsGoodCache[path] = res; - if (!res) printMsg(lvlError, format("path ‘%1%’ is corrupted or missing!") % path); - return res; -} - - -void LocalStore::markContentsGood(const Path & path) -{ - pathContentsGoodCache[path] = true; -} - - -/* Functions for upgrading from the pre-SQLite database. */ - -PathSet LocalStore::queryValidPathsOld() -{ - PathSet paths; - for (auto & i : readDirectory(settings.nixDBPath + "/info")) - if (i.name.at(0) != '.') paths.insert(settings.nixStore + "/" + i.name); - return paths; -} - - -ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) -{ - ValidPathInfo res; - res.path = path; - - /* Read the info file. */ - string baseName = baseNameOf(path); - Path infoFile = (format("%1%/info/%2%") % settings.nixDBPath % baseName).str(); - if (!pathExists(infoFile)) - throw Error(format("path ‘%1%’ is not valid") % path); - string info = readFile(infoFile); - - /* Parse it. */ - Strings lines = tokenizeString<Strings>(info, "\n"); - - for (auto & i : lines) { - string::size_type p = i.find(':'); - if (p == string::npos) - throw Error(format("corrupt line in ‘%1%’: %2%") % infoFile % i); - string name(i, 0, p); - string value(i, p + 2); - if (name == "References") { - Strings refs = tokenizeString<Strings>(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - res.narHash = parseHashField(path, value); - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } - } - - return res; -} - - -/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ -void LocalStore::upgradeStore6() -{ - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - - openDB(true); - - PathSet validPaths = queryValidPathsOld(); - - SQLiteTxn txn(db); - - for (auto & i : validPaths) { - addValidPath(queryPathInfoOld(i), false); - std::cerr << "."; - } - - std::cerr << "|"; - - for (auto & i : validPaths) { - ValidPathInfo info = queryPathInfoOld(i); - unsigned long long referrer = queryValidPathId(i); - for (auto & j : info.references) - addReference(referrer, queryValidPathId(j)); - std::cerr << "."; - } - - std::cerr << "\n"; - - txn.commit(); -} - - #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) @@ -2015,8 +1464,41 @@ void LocalStore::upgradeStore7() void LocalStore::vacuumDB() { - if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "vacuuming SQLite database"); + auto state(_state.lock()); + + if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "vacuuming SQLite database"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + + info->sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, *info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.get("secret-key-files", Strings()); + + for (auto & secretKeyFile : secretKeyFiles) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c7ea9e503..6f2341dec 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,15 +1,14 @@ #pragma once -#include <string> -#include <unordered_set> +#include "sqlite.hh" +#include "pathlocks.hh" #include "store-api.hh" +#include "sync.hh" #include "util.hh" -#include "pathlocks.hh" - -class sqlite3; -class sqlite3_stmt; +#include <string> +#include <unordered_set> namespace nix { @@ -18,8 +17,8 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. */ -const int nixSchemaVersion = 7; + Nix 1.0. Version 7 is Nix 1.3. Version 9 is 1.12. */ +const int nixSchemaVersion = 9; extern string drvsLogDir; @@ -41,54 +40,42 @@ struct OptimiseStats }; -struct RunningSubstituter -{ - Path program; - Pid pid; - AutoCloseFD to, from, error; - FdSource fromBuf; - bool disabled; - RunningSubstituter() : disabled(false) { }; -}; - - -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - class LocalStore : public LocalFSStore { private: - typedef std::map<Path, RunningSubstituter> RunningSubstituters; - RunningSubstituters runningSubstituters; - Path linksDir; + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; - Path reservedPath; + struct State + { + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; + + /* The file to which we write our temporary roots. */ + Path fnTempRoots; + AutoCloseFD fdTempRoots; + }; + + Sync<State, std::recursive_mutex> _state; + + const Path linksDir; + const Path reservedPath; + const Path schemaPath; public: @@ -100,20 +87,18 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPath(const Path & path) override; + std::string getUri() override; + + bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet & paths) override; PathSet queryAllValidPaths() override; - ValidPathInfo queryPathInfo(const Path & path) override; - - Hash queryPathHash(const Path & path) override; + std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override; void queryReferrers(const Path & path, PathSet & referrers) override; - Path queryDeriver(const Path & path) override; - PathSet queryValidDerivers(const Path & path) override; PathSet queryDerivationOutputs(const Path & path) override; @@ -124,9 +109,6 @@ public: PathSet querySubstitutablePaths(const PathSet & paths) override; - void querySubstitutablePathInfos(const Path & substituter, - PathSet & paths, SubstitutablePathInfos & infos); - void querySubstitutablePathInfos(const PathSet & paths, SubstitutablePathInfos & infos) override; @@ -188,90 +170,29 @@ public: void registerValidPaths(const ValidPathInfos & infos); - /* Register that the build of a derivation with output `path' has - failed. */ - void registerFailedPath(const Path & path); - - /* Query whether `path' previously failed to build. */ - bool hasPathFailed(const Path & path); - - PathSet queryFailedPaths() override; - - void clearFailedPaths(const PathSet & paths) override; - void vacuumDB(); /* Repair the contents of the given path by redownloading it using a substituter (if available). */ void repairPath(const Path & path); - /* Check whether the given valid path exists and has the right - contents. */ - bool pathContentsGood(const Path & path); + void addSignatures(const Path & storePath, const StringSet & sigs) override; - void markContentsGood(const Path & path); - - void setSubstituterEnv(); + static bool haveWriteAccess(); private: - Path schemaPath; - - /* Lock file used for upgrading. */ - AutoCloseFD globalLock; - - /* The SQLite database object. */ - SQLite db; - - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtRegisterFailedPath; - SQLiteStmt stmtHasPathFailed; - SQLiteStmt stmtQueryFailedPaths; - SQLiteStmt stmtClearFailedPath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - - /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; - - bool didSetSubstituterEnv; - - /* The file to which we write our temporary roots. */ - Path fnTempRoots; - AutoCloseFD fdTempRoots; - int getSchema(); -public: - - static bool haveWriteAccess(); - -private: - - void openDB(bool create); + void openDB(State & state, bool create); void makeStoreWritable(); - unsigned long long queryValidPathId(const Path & path); - - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); - - void addReference(unsigned long long referrer, unsigned long long reference); + uint64_t queryValidPathId(State & state, const Path & path); - void appendReferrer(const Path & from, const Path & to, bool lock); + uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - - void invalidatePath(const Path & path); + void invalidatePath(State & state, const Path & path); /* Delete a path from the Nix store. */ void invalidatePathChecked(const Path & path); @@ -279,7 +200,7 @@ private: void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths, bool repair, bool & errors); - void updatePathInfo(const ValidPathInfo & info); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); void upgradeStore7(); @@ -307,13 +228,6 @@ private: void removeUnusedLinks(const GCState & state); - void startSubstituter(const Path & substituter, - RunningSubstituter & runningSubstituter); - - string getLineFromSubstituter(RunningSubstituter & run); - - template<class T> T getIntLineFromSubstituter(RunningSubstituter & run); - Path createTempDirInStore(); Path importPath(bool requireSignature, Source & source); @@ -327,8 +241,14 @@ private: void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(const Path & path); - void queryReferrers_(const Path & path, PathSet & referrers); + bool isValidPath_(State & state, const Path & path); + void queryReferrers(State & state, const Path & path, PathSet & referrers); + + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + void signPathInfo(ValidPathInfo & info); + + friend class DerivationGoal; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 9a01596c3..22b0f235e 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -8,7 +8,7 @@ libstore_SOURCES := $(wildcard $(d)/*.cc) libstore_LIBS = libutil libformat -libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) +libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -laws-cpp-sdk-s3 -laws-cpp-sdk-core -pthread ifeq ($(OS), SunOS) libstore_LDFLAGS += -lsocket diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 12472f017..5c284d1b9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -35,12 +35,13 @@ void Store::computeFSClosure(const Path & path, if (includeDerivers && isDerivation(path)) { PathSet outputs = queryDerivationOutputs(path); for (auto & i : outputs) - if (isValidPath(i) && queryDeriver(i) == path) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) edges.insert(i); } } else { - queryReferences(path, edges); + auto info = queryPathInfo(path); + edges = info->references; if (includeOutputs && isDerivation(path)) { PathSet outputs = queryDerivationOutputs(path); @@ -48,10 +49,8 @@ void Store::computeFSClosure(const Path & path, if (isValidPath(i)) edges.insert(i); } - if (includeDerivers) { - Path deriver = queryDeriver(path); - if (isValidPath(deriver)) edges.insert(deriver); - } + if (includeDerivers && isValidPath(info->deriver)) + edges.insert(info->deriver); } for (auto & i : edges) @@ -189,8 +188,10 @@ Paths Store::topoSortPaths(const PathSet & paths) parents.insert(path); PathSet references; - if (isValidPath(path)) - queryReferences(path, references); + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath &) { + } for (auto & i : references) /* Don't traverse into paths that don't exist. That can diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc new file mode 100644 index 000000000..d8b0815bf --- /dev/null +++ b/src/libstore/nar-info-disk-cache.cc @@ -0,0 +1,224 @@ +#include "nar-info-disk-cache.hh" +#include "sync.hh" +#include "sqlite.hh" +#include "globals.hh" + +#include <sqlite3.h> + +namespace nix { + +static const char * schema = R"sql( + +create table if not exists BinaryCaches ( + id integer primary key autoincrement not null, + url text unique not null, + timestamp integer not null, + storeDir text not null, + wantMassQuery integer not null, + priority integer not null +); + +create table if not exists NARs ( + cache integer not null, + hashPart text not null, + namePart text not null, + url text, + compression text, + fileHash text, + fileSize integer, + narHash text, + narSize integer, + refs text, + deriver text, + sigs text, + timestamp integer not null, + primary key (cache, hashPart), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +create table if not exists NARExistence ( + cache integer not null, + storePath text not null, + exist integer not null, + timestamp integer not null, + primary key (cache, storePath), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +)sql"; + +class NarInfoDiskCacheImpl : public NarInfoDiskCache +{ +public: + + /* How long negative lookups are valid. */ + const int ttlNegative = 3600; + + struct State + { + SQLite db; + SQLiteStmt insertCache, queryCache, insertNAR, queryNAR, insertNARExistence, queryNARExistence; + std::map<std::string, int> caches; + }; + + Sync<State> _state; + + NarInfoDiskCacheImpl() + { + auto state(_state.lock()); + + Path dbPath = getCacheDir() + "/nix/binary-cache-v4.sqlite"; + createDirs(dirOf(dbPath)); + + if (sqlite3_open_v2(dbPath.c_str(), &state->db.db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error(format("cannot open store cache ‘%s’") % dbPath); + + if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(state->db, "setting timeout"); + + // We can always reproduce the cache. + if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "making database asynchronous"); + if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "setting journal mode"); + + if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK) + throwSQLiteError(state->db, "initialising database schema"); + + state->insertCache.create(state->db, + "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + + state->queryCache.create(state->db, + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + + state->insertNAR.create(state->db, + "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " + "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + + state->queryNAR.create(state->db, + "select * from NARs where cache = ? and hashPart = ?"); + + state->insertNARExistence.create(state->db, + "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)"); + + state->queryNARExistence.create(state->db, + "select exist, timestamp from NARExistence where cache = ? and storePath = ?"); + } + + int uriToInt(State & state, const std::string & uri) + { + auto i = state.caches.find(uri); + if (i == state.caches.end()) abort(); + return i->second; + } + + void createCache(const std::string & uri) override + { + auto state(_state.lock()); + + // FIXME: race + + state->insertCache.use()(uri)(time(0))(settings.nixStore)(1)(0).exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = sqlite3_last_insert_rowid(state->db); + } + + bool cacheExists(const std::string & uri) override + { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i != state->caches.end()) return true; + + auto queryCache(state->queryCache.use()(uri)); + + if (queryCache.next()) { + state->caches[uri] = queryCache.getInt(0); + return true; + } + + return false; + } + + std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string & uri, const std::string & hashPart) override + { + auto state(_state.lock()); + + auto queryNAR(state->queryNAR.use() + (uriToInt(*state, uri)) + (hashPart)); + + if (!queryNAR.next()) + // FIXME: check NARExistence + return {oUnknown, 0}; + + auto narInfo = make_ref<NarInfo>(); + + // FIXME: implement TTL. + + auto namePart = queryNAR.getStr(2); + narInfo->path = settings.nixStore + "/" + + hashPart + (namePart.empty() ? "" : "-" + namePart); + narInfo->url = queryNAR.getStr(3); + narInfo->compression = queryNAR.getStr(4); + if (!queryNAR.isNull(5)) + narInfo->fileHash = parseHash(queryNAR.getStr(5)); + narInfo->fileSize = queryNAR.getInt(6); + narInfo->narHash = parseHash(queryNAR.getStr(7)); + narInfo->narSize = queryNAR.getInt(8); + for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " ")) + narInfo->references.insert(settings.nixStore + "/" + r); + if (!queryNAR.isNull(10)) + narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(10); + for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " ")) + narInfo->sigs.insert(sig); + + return {oValid, narInfo}; + } + + void upsertNarInfo( + const std::string & uri, const std::string & hashPart, + std::shared_ptr<ValidPathInfo> info) override + { + auto state(_state.lock()); + + if (info) { + + auto narInfo = std::dynamic_pointer_cast<NarInfo>(info); + + assert(hashPart == storePathToHash(info->path)); + + state->insertNAR.use() + (uriToInt(*state, uri)) + (hashPart) + (storePathToName(info->path)) + (narInfo ? narInfo->url : "", narInfo != 0) + (narInfo ? narInfo->compression : "", narInfo != 0) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) + (info->narHash.to_string()) + (info->narSize) + (concatStringsSep(" ", info->shortRefs())) + (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") + (concatStringsSep(" ", info->sigs)) + (time(0)).exec(); + + } else { + // not implemented + abort(); + } + } +}; + +ref<NarInfoDiskCache> getNarInfoDiskCache() +{ + static Sync<std::shared_ptr<NarInfoDiskCache>> cache; + + auto cache_(cache.lock()); + if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>(); + return ref<NarInfoDiskCache>(*cache_); +} + +} diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh new file mode 100644 index 000000000..f4e3fbbdc --- /dev/null +++ b/src/libstore/nar-info-disk-cache.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "ref.hh" +#include "nar-info.hh" + +namespace nix { + +class NarInfoDiskCache +{ +public: + typedef enum { oValid, oInvalid, oUnknown } Outcome; + + virtual void createCache(const std::string & uri) = 0; + + virtual bool cacheExists(const std::string & uri) = 0; + + virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string & uri, const std::string & hashPart) = 0; + + virtual void upsertNarInfo( + const std::string & uri, const std::string & hashPart, + std::shared_ptr<ValidPathInfo> info) = 0; +}; + +/* Return a singleton cache object that can be used concurrently by + multiple threads. */ +ref<NarInfoDiskCache> getNarInfoDiskCache(); + +} diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index e9260a09b..c0c5cecd1 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,4 +1,3 @@ -#include "crypto.hh" #include "globals.hh" #include "nar-info.hh" @@ -6,16 +5,16 @@ namespace nix { NarInfo::NarInfo(const std::string & s, const std::string & whence) { - auto corrupt = [&]() { + auto corrupt = [&]() [[noreturn]] { throw Error("NAR info file ‘%1%’ is corrupt"); }; auto parseHashField = [&](const string & s) { - string::size_type colon = s.find(':'); - if (colon == string::npos) corrupt(); - HashType ht = parseHashType(string(s, 0, colon)); - if (ht == htUnknown) corrupt(); - return parseHash16or32(ht, string(s, colon + 1)); + try { + return parseHash(s); + } catch (BadHash &) { + corrupt(); + } }; size_t pos = 0; @@ -66,7 +65,7 @@ NarInfo::NarInfo(const std::string & s, const std::string & whence) else if (name == "System") system = value; else if (name == "Sig") - sig = value; + sigs.insert(value); pos = eol + 1; } @@ -98,37 +97,10 @@ std::string NarInfo::to_string() const if (!system.empty()) res += "System: " + system + "\n"; - if (!sig.empty()) + for (auto sig : sigs) res += "Sig: " + sig + "\n"; return res; } -std::string NarInfo::fingerprint() const -{ - return - "1;" + path + ";" - + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" - + std::to_string(narSize) + ";" - + concatStringsSep(",", references); -} - -Strings NarInfo::shortRefs() const -{ - Strings refs; - for (auto & r : references) - refs.push_back(baseNameOf(r)); - return refs; -} - -void NarInfo::sign(const SecretKey & secretKey) -{ - sig = secretKey.signDetached(fingerprint()); -} - -bool NarInfo::checkSignature(const PublicKeys & publicKeys) const -{ - return sig != "" && verifyDetached(fingerprint(), sig, publicKeys); -} - } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 22e27cb42..6bc2f03b1 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -13,31 +13,12 @@ struct NarInfo : ValidPathInfo Hash fileHash; uint64_t fileSize = 0; std::string system; - std::string sig; // FIXME: support multiple signatures NarInfo() { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const std::string & s, const std::string & whence); std::string to_string() const; - - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint() const; - - void sign(const SecretKey & secretKey); - - /* Return true iff this .narinfo is signed by one of the specified - keys. */ - bool checkSignature(const PublicKeys & publicKeys) const; - -private: - - Strings shortRefs() const; }; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 23cbe7e26..ad7fe0e8b 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -228,7 +228,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) for (auto & i : paths) { addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ - startNest(nest, lvlChatty, format("hashing files in ‘%1%’") % i); + Activity act(*logger, lvlChatty, format("hashing files in ‘%1%’") % i); optimisePath_(stats, i, inodeHash); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 82b7cfd7c..5a254a610 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -49,6 +49,12 @@ RemoteStore::RemoteStore(size_t maxConnections) } +std::string RemoteStore::getUri() +{ + return "daemon"; +} + + ref<RemoteStore::Connection> RemoteStore::openConnection() { auto conn = make_ref<Connection>(); @@ -61,27 +67,15 @@ ref<RemoteStore::Connection> RemoteStore::openConnection() string socketPath = settings.nixDaemonSocketFile; - /* Urgh, sockaddr_un allows path names of only 108 characters. So - chdir to the socket directory so that we can pass a relative - path name. !!! this is probably a bad idea in multi-threaded - applications... */ - AutoCloseFD fdPrevDir = open(".", O_RDONLY); - if (fdPrevDir == -1) throw SysError("couldn't open current directory"); - if (chdir(dirOf(socketPath).c_str()) == -1) throw SysError(format("couldn't change to directory of ‘%1%’") % socketPath); - Path socketPathRel = "./" + baseNameOf(socketPath); - struct sockaddr_un addr; addr.sun_family = AF_UNIX; - if (socketPathRel.size() >= sizeof(addr.sun_path)) - throw Error(format("socket path ‘%1%’ is too long") % socketPathRel); - strcpy(addr.sun_path, socketPathRel.c_str()); + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket path ‘%1%’ is too long") % socketPath); + strcpy(addr.sun_path, socketPath.c_str()); if (connect(conn->fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError(format("cannot connect to daemon at ‘%1%’") % socketPath); - if (fchdir(fdPrevDir) == -1) - throw SysError("couldn't change back to previous directory"); - conn->from.fd = conn->fd; conn->to.fd = conn->fd; @@ -132,9 +126,9 @@ void RemoteStore::setOptions(ref<Connection> conn) if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 2) conn->to << settings.useBuildHook; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 4) - conn->to << settings.buildVerbosity - << logType - << settings.printBuildTrace; + conn->to << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 6) conn->to << settings.buildCores; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 10) @@ -153,7 +147,7 @@ void RemoteStore::setOptions(ref<Connection> conn) } -bool RemoteStore::isValidPath(const Path & path) +bool RemoteStore::isValidPathUncached(const Path & path) { auto conn(connections->get()); conn->to << wopIsValidPath << path; @@ -251,44 +245,38 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } -ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +std::shared_ptr<ValidPathInfo> RemoteStore::queryPathInfoUncached(const Path & path) { auto conn(connections->get()); conn->to << wopQueryPathInfo << path; - conn->processStderr(); - ValidPathInfo info; - info.path = path; - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.narHash = parseHash(htSHA256, readString(conn->from)); - info.references = readStorePaths<PathSet>(conn->from); - info.registrationTime = readInt(conn->from); - info.narSize = readLongLong(conn->from); + try { + conn->processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.what()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid = readInt(conn->from) != 0; + if (!valid) throw InvalidPath(format("path ‘%s’ is not valid") % path); + } + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = readString(conn->from); + if (info->deriver != "") assertStorePath(info->deriver); + info->narHash = parseHash(htSHA256, readString(conn->from)); + info->references = readStorePaths<PathSet>(conn->from); + info->registrationTime = readInt(conn->from); + info->narSize = readLongLong(conn->from); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + info->ultimate = readInt(conn->from) != 0; + info->sigs = readStrings<StringSet>(conn->from); + } return info; } -Hash RemoteStore::queryPathHash(const Path & path) -{ - auto conn(connections->get()); - conn->to << wopQueryPathHash << path; - conn->processStderr(); - string hash = readString(conn->from); - return parseHash(htSHA256, hash); -} - - -void RemoteStore::queryReferences(const Path & path, - PathSet & references) -{ - auto conn(connections->get()); - conn->to << wopQueryReferences << path; - conn->processStderr(); - PathSet references2 = readStorePaths<PathSet>(conn->from); - references.insert(references2.begin(), references2.end()); -} - - void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) { @@ -300,17 +288,6 @@ void RemoteStore::queryReferrers(const Path & path, } -Path RemoteStore::queryDeriver(const Path & path) -{ - auto conn(connections->get()); - conn->to << wopQueryDeriver << path; - conn->processStderr(); - Path drvPath = readString(conn->from); - if (drvPath != "") assertStorePath(drvPath); - return drvPath; -} - - PathSet RemoteStore::queryValidDerivers(const Path & path) { auto conn(connections->get()); @@ -525,26 +502,14 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths = readStrings<PathSet>(conn->from); results.bytesFreed = readLongLong(conn->from); readLongLong(conn->from); // obsolete -} - -PathSet RemoteStore::queryFailedPaths() -{ - auto conn(connections->get()); - conn->to << wopQueryFailedPaths; - conn->processStderr(); - return readStorePaths<PathSet>(conn->from); + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } } -void RemoteStore::clearFailedPaths(const PathSet & paths) -{ - auto conn(connections->get()); - conn->to << wopClearFailedPaths << paths; - conn->processStderr(); - readInt(conn->from); -} - void RemoteStore::optimiseStore() { auto conn(connections->get()); @@ -553,6 +518,7 @@ void RemoteStore::optimiseStore() readInt(conn->from); } + bool RemoteStore::verifyStore(bool checkContents, bool repair) { auto conn(connections->get()); @@ -562,6 +528,15 @@ bool RemoteStore::verifyStore(bool checkContents, bool repair) } +void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + auto conn(connections->get()); + conn->to << wopAddSignatures << storePath << sigs; + conn->processStderr(); + readInt(conn->from); +} + + RemoteStore::Connection::~Connection() { try { @@ -592,10 +567,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) writeString(buf, source->read(buf, len), to); to.flush(); } - else { - string s = readString(from); - writeToStderr(s); - } + else + printMsg(lvlError, chomp(readString(from))); } if (msg == STDERR_ERROR) { string error = readString(from); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 85c8292c7..8e45a7449 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -26,22 +26,18 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPath(const Path & path) override; + std::string getUri() override; + + bool isValidPathUncached(const Path & path) override; PathSet queryValidPaths(const PathSet & paths) override; PathSet queryAllValidPaths() override; - ValidPathInfo queryPathInfo(const Path & path) override; - - Hash queryPathHash(const Path & path) override; - - void queryReferences(const Path & path, PathSet & references) override; + std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) override; void queryReferrers(const Path & path, PathSet & referrers) override; - Path queryDeriver(const Path & path) override; - PathSet queryValidDerivers(const Path & path) override; PathSet queryDerivationOutputs(const Path & path) override; @@ -85,14 +81,12 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - PathSet queryFailedPaths() override; - - void clearFailedPaths(const PathSet & paths) override; - void optimiseStore() override; bool verifyStore(bool checkContents, bool repair) override; + void addSignatures(const Path & storePath, const StringSet & sigs) override; + private: struct Connection diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000..cffcb1bf2 --- /dev/null +++ b/src/libstore/s3-binary-cache-store.cc @@ -0,0 +1,253 @@ +#include "s3-binary-cache-store.hh" +#include "nar-info.hh" +#include "nar-info-disk-cache.hh" +#include "globals.hh" + +#include <aws/core/client/ClientConfiguration.h> +#include <aws/s3/S3Client.h> +#include <aws/s3/model/CreateBucketRequest.h> +#include <aws/s3/model/GetBucketLocationRequest.h> +#include <aws/s3/model/GetObjectRequest.h> +#include <aws/s3/model/HeadObjectRequest.h> +#include <aws/s3/model/PutObjectRequest.h> +#include <aws/s3/model/ListObjectsRequest.h> + +namespace nix { + +struct S3Error : public Error +{ + Aws::S3::S3Errors err; + S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) + : Error(fs), err(err) { }; +}; + +/* Helper: given an Outcome<R, E>, return R in case of success, or + throw an exception in case of an error. */ +template<typename R, typename E> +R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome<R, E> && outcome) +{ + if (!outcome.IsSuccess()) + throw S3Error( + outcome.GetError().GetErrorType(), + fs.s + ": " + outcome.GetError().GetMessage()); + return outcome.GetResultWithOwnership(); +} + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +{ + std::string bucketName; + + ref<Aws::Client::ClientConfiguration> config; + ref<Aws::S3::S3Client> client; + + Stats stats; + + S3BinaryCacheStoreImpl(std::shared_ptr<Store> localStore, + const StoreParams & params, const std::string & bucketName) + : S3BinaryCacheStore(localStore, params) + , bucketName(bucketName) + , config(makeConfig()) + , client(make_ref<Aws::S3::S3Client>(*config)) + { + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() + { + return "s3://" + bucketName; + } + + ref<Aws::Client::ClientConfiguration> makeConfig() + { + auto res = make_ref<Aws::Client::ClientConfiguration>(); + res->region = Aws::Region::US_EAST_1; // FIXME: make configurable + res->requestTimeoutMs = 600 * 1000; + return res; + } + + void init() + { + if (!diskCache->cacheExists(getUri())) { + + /* Create the bucket if it doesn't already exists. */ + // FIXME: HeadBucket would be more appropriate, but doesn't return + // an easily parsed 404 message. + auto res = client->GetBucketLocation( + Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); + + if (!res.IsSuccess()) { + if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) + throw Error(format("AWS error checking bucket ‘%s’: %s") % bucketName % res.GetError().GetMessage()); + + checkAws(format("AWS error creating bucket ‘%s’") % bucketName, + client->CreateBucket( + Aws::S3::Model::CreateBucketRequest() + .WithBucket(bucketName) + .WithCreateBucketConfiguration( + Aws::S3::Model::CreateBucketConfiguration() + /* .WithLocationConstraint( + Aws::S3::Model::BucketLocationConstraint::US) */ ))); + } + + BinaryCacheStore::init(); + + diskCache->createCache(getUri()); + } + } + + const Stats & getS3Stats() + { + return stats; + } + + /* This is a specialisation of isValidPath() that optimistically + fetches the .narinfo file, rather than first checking for its + existence via a HEAD request. Since .narinfos are small, doing + a GET is unlikely to be slower than HEAD. */ + bool isValidPathUncached(const Path & storePath) + { + try { + queryPathInfo(storePath); + return true; + } catch (InvalidPath & e) { + return false; + } + } + + bool fileExists(const std::string & path) + { + stats.head++; + + auto res = client->HeadObject( + Aws::S3::Model::HeadObjectRequest() + .WithBucket(bucketName) + .WithKey(path)); + + if (!res.IsSuccess()) { + auto & error = res.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN // FIXME + && error.GetMessage().find("404") != std::string::npos) + return false; + throw Error(format("AWS error fetching ‘%s’: %s") % path % error.GetMessage()); + } + + return true; + } + + void upsertFile(const std::string & path, const std::string & data) + { + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + auto stream = std::make_shared<std::stringstream>(data); + + request.SetBody(stream); + + stats.put++; + stats.putBytes += data.size(); + + auto now1 = std::chrono::steady_clock::now(); + + auto result = checkAws(format("AWS error uploading ‘%s’") % path, + client->PutObject(request)); + + auto now2 = std::chrono::steady_clock::now(); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + + printMsg(lvlInfo, format("uploaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") + % bucketName % path % data.size() % duration); + + stats.putTimeMs += duration; + } + + std::shared_ptr<std::string> getFile(const std::string & path) + { + debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path); + + auto request = + Aws::S3::Model::GetObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetResponseStreamFactory([&]() { + return Aws::New<std::stringstream>("STRINGSTREAM"); + }); + + stats.get++; + + try { + + auto now1 = std::chrono::steady_clock::now(); + + auto result = checkAws(format("AWS error fetching ‘%s’") % path, + client->GetObject(request)); + + auto now2 = std::chrono::steady_clock::now(); + + auto res = dynamic_cast<std::stringstream &>(result.GetBody()).str(); + + auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); + + printMsg(lvlTalkative, format("downloaded ‘s3://%1%/%2%’ (%3% bytes) in %4% ms") + % bucketName % path % res.size() % duration); + + stats.getBytes += res.size(); + stats.getTimeMs += duration; + + return std::make_shared<std::string>(res); + + } catch (S3Error & e) { + if (e.err == Aws::S3::S3Errors::NO_SUCH_KEY) return 0; + throw; + } + } + + PathSet queryAllValidPaths() override + { + PathSet paths; + std::string marker; + + do { + debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker); + + auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName, + client->ListObjects( + Aws::S3::Model::ListObjectsRequest() + .WithBucket(bucketName) + .WithDelimiter("/") + .WithMarker(marker))); + + auto & contents = res.GetContents(); + + debug(format("got %d keys, next marker ‘%s’") + % contents.size() % res.GetNextMarker()); + + for (auto object : contents) { + auto & key = object.GetKey(); + if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; + paths.insert(settings.nixStore + "/" + key.substr(0, key.size() - 8)); + } + + marker = res.GetNextMarker(); + } while (!marker.empty()); + + return paths; + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ + if (std::string(uri, 0, 5) != "s3://") return 0; + auto store = std::make_shared<S3BinaryCacheStoreImpl>(std::shared_ptr<Store>(0), + params, std::string(uri, 5)); + store->init(); + return store; +}); + +} diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh new file mode 100644 index 000000000..2751a9d01 --- /dev/null +++ b/src/libstore/s3-binary-cache-store.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "binary-cache-store.hh" + +#include <atomic> + +namespace nix { + +class S3BinaryCacheStore : public BinaryCacheStore +{ +protected: + + S3BinaryCacheStore(std::shared_ptr<Store> localStore, + const StoreParams & params) + : BinaryCacheStore(localStore, params) + { } + +public: + + struct Stats + { + std::atomic<uint64_t> put{0}; + std::atomic<uint64_t> putBytes{0}; + std::atomic<uint64_t> putTimeMs{0}; + std::atomic<uint64_t> get{0}; + std::atomic<uint64_t> getBytes{0}; + std::atomic<uint64_t> getTimeMs{0}; + std::atomic<uint64_t> head{0}; + }; + + const Stats & getS3Stats(); +}; + +} diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index c1b4a689a..91878af15 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -4,7 +4,9 @@ create table if not exists ValidPaths ( hash text not null, registrationTime integer not null, deriver text, - narSize integer + narSize integer, + ultimate integer, -- null implies "false" + sigs text -- space-separated ); create table if not exists Refs ( @@ -37,8 +39,3 @@ create table if not exists DerivationOutputs ( ); create index if not exists IndexDerivationOutputs on DerivationOutputs(path); - -create table if not exists FailedPaths ( - path text primary key not null, - time integer not null -); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc new file mode 100644 index 000000000..816f9984d --- /dev/null +++ b/src/libstore/sqlite.cc @@ -0,0 +1,172 @@ +#include "sqlite.hh" +#include "util.hh" + +#include <sqlite3.h> + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() +{ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; +} + +int SQLiteStmt::Use::step() +{ + return sqlite3_step(stmt); +} + +void SQLiteStmt::Use::exec() +{ + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +bool SQLiteStmt::Use::isNull(int col) +{ + return sqlite3_column_type(stmt, col) == SQLITE_NULL; +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh new file mode 100644 index 000000000..d6b4a8d91 --- /dev/null +++ b/src/libstore/sqlite.hh @@ -0,0 +1,103 @@ +#pragma once + +#include <functional> +#include <string> + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } + void create(sqlite3 * db, const std::string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + bool isNull(int col); + }; + + Use use() + { + return Use(*this); + } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template<typename T> +T retrySQLite(std::function<T()> fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} + +} diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 378233654..463e132e0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,8 @@ -#include "store-api.hh" +#include "crypto.hh" #include "globals.hh" +#include "store-api.hh" #include "util.hh" +#include "nar-info-disk-cache.hh" namespace nix { @@ -15,6 +17,7 @@ bool isInStore(const Path & path) bool isStorePath(const Path & path) { return isInStore(path) + && path.size() >= settings.nixStore.size() + 1 + storePathHashLen && path.find('/', settings.nixStore.size() + 1) == Path::npos; } @@ -61,13 +64,16 @@ Path followLinksToStorePath(const Path & path) string storePathToName(const Path & path) { assertStorePath(path); - return string(path, settings.nixStore.size() + storePathHashLen + 2); + auto l = settings.nixStore.size() + 1 + storePathHashLen; + assert(path.size() >= l); + return path.size() == l ? "" : string(path, l + 1); } string storePathToHash(const Path & path) { assertStorePath(path); + assert(path.size() >= settings.nixStore.size() + 1 + storePathHashLen); return string(path, settings.nixStore.size() + 1, storePathHashLen); } @@ -135,14 +141,14 @@ void checkStoreName(const string & name) if <type> = "source": the serialisation of the path from which this store path is copied, as returned by hashPath() - if <type> = "output:out": + if <type> = "output:<id>": for non-fixed derivation outputs: the derivation (see hashDerivationModulo() in primops.cc) for paths copied by addToStore() or produced by fixed-output derivations: the string "fixed:out:<rec><algo>:<hash>:", where - <rec> = "r:" for recursive (path) hashes, or "" or flat + <rec> = "r:" for recursive (path) hashes, or "" for flat (file) hashes <algo> = "md5", "sha1" or "sha256" <hash> = base-16 representation of the path or flat hash of @@ -224,10 +230,89 @@ Path computeStorePathForText(const string & name, const string & s, } -void Store::queryReferences(const Path & path, PathSet & references) +std::string Store::getUri() +{ + return ""; +} + + +bool Store::isValidPath(const Path & storePath) +{ + auto hashPart = storePathToHash(storePath); + + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + return *res != 0; + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + return res.first == NarInfoDiskCache::oValid; + } + } + + return isValidPathUncached(storePath); + + // FIXME: insert result into NARExistence table of diskCache. +} + + +ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) { - ValidPathInfo info = queryPathInfo(path); - references.insert(info.references.begin(), info.references.end()); + auto hashPart = storePathToHash(storePath); + + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + if (!*res) + throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + return ref<ValidPathInfo>(*res); + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + if (res.first == NarInfoDiskCache::oInvalid || + (res.second->path != storePath && storePathToName(storePath) != "")) + throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + return ref<ValidPathInfo>(res.second); + } + } + + auto info = queryPathInfoUncached(storePath); + + if (diskCache && info) + diskCache->upsertNarInfo(getUri(), hashPart, info); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } + + if (!info + || (info->path != storePath && storePathToName(storePath) != "")) + { + stats.narInfoMissing++; + throw InvalidPath(format("path ‘%s’ is not valid") % storePath); + } + + return ref<ValidPathInfo>(info); } @@ -242,19 +327,19 @@ string Store::makeValidityRegistration(const PathSet & paths, for (auto & i : paths) { s += i + "\n"; - ValidPathInfo info = queryPathInfo(i); + auto info = queryPathInfo(i); if (showHash) { - s += printHash(info.narHash) + "\n"; - s += (format("%1%\n") % info.narSize).str(); + s += printHash(info->narHash) + "\n"; + s += (format("%1%\n") % info->narSize).str(); } - Path deriver = showDerivers ? info.deriver : ""; + Path deriver = showDerivers ? info->deriver : ""; s += deriver + "\n"; - s += (format("%1%\n") % info.references.size()).str(); + s += (format("%1%\n") % info->references.size()).str(); - for (auto & j : info.references) + for (auto & j : info->references) s += j + "\n"; } @@ -262,6 +347,16 @@ string Store::makeValidityRegistration(const PathSet & paths, } +const Store::Stats & Store::getStats() +{ + { + auto state_(state.lock()); + stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } + return stats; +} + + ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) { ValidPathInfo info; @@ -309,6 +404,50 @@ void Store::exportPaths(const Paths & paths, } +std::string ValidPathInfo::fingerprint() const +{ + if (narSize == 0 || !narHash) + throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known") + % path); + return + "1;" + path + ";" + + printHashType(narHash.type) + ":" + printHash32(narHash) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", references); +} + + +void ValidPathInfo::sign(const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint())); +} + + +unsigned int ValidPathInfo::checkSignatures(const PublicKeys & publicKeys) const +{ + unsigned int good = 0; + for (auto & sig : sigs) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(baseNameOf(r)); + return refs; +} + + } @@ -322,10 +461,22 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; -ref<Store> openStoreAt(const std::string & uri) +ref<Store> openStoreAt(const std::string & uri_) { + auto uri(uri_); + StoreParams params; + auto q = uri.find('?'); + if (q != std::string::npos) { + for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { + auto e = s.find('='); + if (e != std::string::npos) + params[s.substr(0, e)] = s.substr(e + 1); + } + uri = uri_.substr(0, q); + } + for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri); + auto store = fun(uri, params); if (store) return ref<Store>(store); } @@ -339,7 +490,10 @@ ref<Store> openStore() } -static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> { +static RegisterStoreImplementation regStore([]( + const std::string & uri, const StoreParams & params) + -> std::shared_ptr<Store> +{ enum { mDaemon, mLocal, mAuto } mode; if (uri == "daemon") mode = mDaemon; @@ -362,4 +516,39 @@ static RegisterStoreImplementation regStore([](const std::string & uri) -> std:: }); +std::list<ref<Store>> getDefaultSubstituters() +{ + struct State { + bool done = false; + std::list<ref<Store>> stores; + }; + static Sync<State> state_; + + auto state(state_.lock()); + + if (state->done) return state->stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (done.count(uri)) return; + done.insert(uri); + state->stores.push_back(openStoreAt(uri)); + }; + + for (auto uri : settings.get("substituters", Strings())) + addStore(uri); + + for (auto uri : settings.get("binary-caches", Strings())) + addStore(uri); + + for (auto uri : settings.get("extra-binary-caches", Strings())) + addStore(uri); + + state->done = true; + + return state->stores; +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 72b0b0f03..29685c9d1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,11 +2,15 @@ #include "hash.hh" #include "serialise.hh" +#include "crypto.hh" +#include "lru-cache.hh" +#include "sync.hh" -#include <string> +#include <atomic> #include <limits> #include <map> #include <memory> +#include <string> namespace nix { @@ -95,8 +99,15 @@ struct ValidPathInfo Hash narHash; PathSet references; time_t registrationTime = 0; - unsigned long long narSize = 0; // 0 = unknown - unsigned long long id; // internal use only + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it was built + locally or is content-addressable (e.g. added via addToStore() + or the result of a fixed-output derivation). */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified bool operator == (const ValidPathInfo & i) const { @@ -105,6 +116,27 @@ struct ValidPathInfo && narHash == i.narHash && references == i.references; } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey & secretKey); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys. */ + unsigned int checkSignatures(const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + + Strings shortRefs() const; + + virtual ~ValidPathInfo() { } }; typedef list<ValidPathInfo> ValidPathInfos; @@ -123,7 +155,6 @@ struct BuildResult InputRejected, OutputRejected, TransientFailure, // possibly transient - CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -141,42 +172,62 @@ struct BuildResult struct BasicDerivation; struct Derivation; class FSAccessor; +class NarInfoDiskCache; class Store : public std::enable_shared_from_this<Store> { +protected: + + struct State + { + LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024}; + }; + + Sync<State> state; + + std::shared_ptr<NarInfoDiskCache> diskCache; + public: virtual ~Store() { } + virtual std::string getUri() = 0; + /* Check whether a path is valid. */ - virtual bool isValidPath(const Path & path) = 0; + bool isValidPath(const Path & path); + +protected: + + virtual bool isValidPathUncached(const Path & path) = 0; + +public: /* Query which of the given paths is valid. */ virtual PathSet queryValidPaths(const PathSet & paths) = 0; - /* Query the set of all valid paths. */ + /* Query the set of all valid paths. Note that for some store + backends, the name part of store paths may be omitted + (i.e. you'll get /nix/store/<hash> rather than + /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the + full store path. */ virtual PathSet queryAllValidPaths() = 0; - /* Query information about a valid path. */ - virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + /* Query information about a valid path. It is permitted to omit + the name part of the store path. */ + ref<const ValidPathInfo> queryPathInfo(const Path & path); - /* Query the hash of a valid path. */ - virtual Hash queryPathHash(const Path & path) = 0; +protected: - /* Query the set of outgoing FS references for a store path. The - result is not cleared. */ - virtual void queryReferences(const Path & path, PathSet & references); + virtual std::shared_ptr<ValidPathInfo> queryPathInfoUncached(const Path & path) = 0; + +public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const Path & path, PathSet & referrers) = 0; - /* Query the deriver of a store path. Return the empty string if - no deriver has been set. */ - virtual Path queryDeriver(const Path & path) = 0; - /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the derivation that was actually used to produce `path', which may @@ -215,6 +266,9 @@ public: virtual Path addTextToStore(const string & name, const string & s, const PathSet & references, bool repair = false) = 0; + /* Write a NAR dump of a store path. */ + virtual void narFromPath(const Path & path, Sink & sink) = 0; + /* Export a store path, that is, create a NAR dump of the store path and append its references and its deriver. Optionally, a cryptographic signature (created by OpenSSL) of the preceding @@ -297,13 +351,6 @@ public: /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; - /* Return the set of paths that have failed to build.*/ - virtual PathSet queryFailedPaths() = 0; - - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ - virtual void clearFailedPaths(const PathSet & paths) = 0; - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ @@ -321,6 +368,10 @@ public: /* Return an object to access files in the Nix store. */ virtual ref<FSAccessor> getFSAccessor() = 0; + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + /* Utility functions. */ /* Read a derivation, after ensuring its existence through @@ -349,11 +400,36 @@ public: relation. If p refers to q, then p preceeds q in this list. */ Paths topoSortPaths(const PathSet & paths); + struct Stats + { + std::atomic<uint64_t> narInfoRead{0}; + std::atomic<uint64_t> narInfoReadAverted{0}; + std::atomic<uint64_t> narInfoMissing{0}; + std::atomic<uint64_t> narInfoWrite{0}; + std::atomic<uint64_t> pathInfoCacheSize{0}; + std::atomic<uint64_t> narRead{0}; + std::atomic<uint64_t> narReadBytes{0}; + std::atomic<uint64_t> narReadCompressedBytes{0}; + std::atomic<uint64_t> narWrite{0}; + std::atomic<uint64_t> narWriteAverted{0}; + std::atomic<uint64_t> narWriteBytes{0}; + std::atomic<uint64_t> narWriteCompressedBytes{0}; + std::atomic<uint64_t> narWriteCompressionTimeMs{0}; + }; + + const Stats & getStats(); + +protected: + + Stats stats; + }; class LocalFSStore : public Store { +public: + void narFromPath(const Path & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; }; @@ -453,12 +529,17 @@ ref<Store> openStoreAt(const std::string & uri); ref<Store> openStore(); -ref<Store> openLocalBinaryCacheStore(std::shared_ptr<Store> localStore, - const Path & secretKeyFile, const Path & binaryCacheDir); +/* Return the default substituter stores, defined by the + ‘substituters’ option and various legacy options like + ‘binary-caches’. */ +std::list<ref<Store>> getDefaultSubstituters(); /* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>(const std::string & uri)> OpenStore; +typedef std::map<std::string, std::string> StoreParams; + +typedef std::function<std::shared_ptr<Store>( + const std::string & uri, const StoreParams & params)> OpenStore; struct RegisterStoreImplementation { @@ -485,6 +566,7 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, MakeError(SubstError, Error) MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 7d9bcb58a..d133328d1 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10f +#define PROTOCOL_VERSION 0x111 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -14,8 +14,8 @@ namespace nix { typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, - wopQueryPathHash = 4, - wopQueryReferences = 5, + wopQueryPathHash = 4, // obsolete + wopQueryReferences = 5, // obsolete wopQueryReferrers = 6, wopAddToStore = 7, wopAddTextToStore = 8, @@ -26,7 +26,7 @@ typedef enum { wopSyncWithGC = 13, wopFindRoots = 14, wopExportPath = 16, - wopQueryDeriver = 18, + wopQueryDeriver = 18, // obsolete wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, @@ -45,6 +45,7 @@ typedef enum { wopOptimiseStore = 34, wopVerifyStore = 35, wopBuildDerivation = 36, + wopAddSignatures = 37, } WorkerOp; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 6e4b82a27..115484f9e 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -71,10 +71,11 @@ void Args::printHelp(const string & programName, std::ostream & out) void Args::printFlags(std::ostream & out) { Table2 table; - for (auto & flags : longFlags) + for (auto & flag : longFlags) table.push_back(std::make_pair( - "--" + flags.first + renderLabels(flags.second.labels), - flags.second.description)); + (flag.second.shortName ? std::string("-") + flag.second.shortName + ", " : " ") + + "--" + flag.first + renderLabels(flag.second.labels), + flag.second.description)); printTable(out, table); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 4469a046d..6aa08aaca 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -29,6 +29,7 @@ protected: /* Flags. */ struct Flag { + char shortName; std::string description; Strings labels; size_t arity; @@ -63,7 +64,7 @@ public: const Strings & labels, const std::string & description, size_t arity, std::function<void(Strings)> handler) { - auto flag = Flag{description, labels, arity, handler}; + auto flag = Flag{shortName, description, labels, arity, handler}; if (shortName) shortFlags[shortName] = flag; longFlags[longName] = flag; } diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index a3fa0dab7..4d15d2acd 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,88 +1,88 @@ #include "compression.hh" -#include "types.hh" +#include "util.hh" +#include "finally.hh" #include <lzma.h> +#include <bzlib.h> #include <cstdio> +#include <cstring> namespace nix { -/* RAII wrapper around lzma_stream. */ -struct LzmaStream +static ref<std::string> compressXZ(const std::string & in) { lzma_stream strm; - LzmaStream() : strm(LZMA_STREAM_INIT) { }; - ~LzmaStream() { lzma_end(&strm); }; - lzma_stream & operator()() { return strm; } -}; - -std::string compressXZ(const std::string & in) -{ - LzmaStream strm; // FIXME: apply the x86 BCJ filter? lzma_ret ret = lzma_easy_encoder( - &strm(), 6, LZMA_CHECK_CRC64); + &strm, 6, LZMA_CHECK_CRC64); if (ret != LZMA_OK) throw Error("unable to initialise lzma encoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; - string res; - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { + checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) return res; if (ret != LZMA_OK) - throw Error("error while decompressing xz file"); + throw Error("error while compressing xz file"); } } -std::string decompressXZ(const std::string & in) +static ref<std::string> decompressXZ(const std::string & in) { - LzmaStream strm; + lzma_stream strm; lzma_ret ret = lzma_stream_decoder( - &strm(), UINT64_MAX, LZMA_CONCATENATED); + &strm, UINT64_MAX, LZMA_CONCATENATED); if (ret != LZMA_OK) throw Error("unable to initialise lzma decoder"); + Finally free([&]() { lzma_end(&strm); }); + lzma_action action = LZMA_RUN; uint8_t outbuf[BUFSIZ]; - string res; - strm().next_in = (uint8_t *) in.c_str(); - strm().avail_in = in.size(); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (uint8_t *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); while (true) { + checkInterrupt(); - if (strm().avail_in == 0) + if (strm.avail_in == 0) action = LZMA_FINISH; - lzma_ret ret = lzma_code(&strm(), action); + lzma_ret ret = lzma_code(&strm, action); - if (strm().avail_out == 0 || ret == LZMA_STREAM_END) { - res.append((char *) outbuf, sizeof(outbuf) - strm().avail_out); - strm().next_out = outbuf; - strm().avail_out = sizeof(outbuf); + if (strm.avail_out == 0 || ret == LZMA_STREAM_END) { + res->append((char *) outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); } if (ret == LZMA_STREAM_END) @@ -93,4 +93,108 @@ std::string decompressXZ(const std::string & in) } } +static ref<std::string> compressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 encoder"); + + Finally free([&]() { BZ2_bzCompressEnd(&strm); }); + + int action = BZ_RUN; + char outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + if (strm.avail_in == 0) + action = BZ_FINISH; + + int ret = BZ2_bzCompress(&strm, action); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK && ret != BZ_FINISH_OK) + Error("error while compressing bzip2 file"); + } + + return res; +} + +static ref<std::string> decompressBzip2(const std::string & in) +{ + bz_stream strm; + memset(&strm, 0, sizeof(strm)); + + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw Error("unable to initialise bzip2 decoder"); + + Finally free([&]() { BZ2_bzDecompressEnd(&strm); }); + + char outbuf[BUFSIZ]; + ref<std::string> res = make_ref<std::string>(); + strm.next_in = (char *) in.c_str(); + strm.avail_in = in.size(); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + + while (true) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + + if (strm.avail_out == 0 || ret == BZ_STREAM_END) { + res->append(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + if (ret == BZ_STREAM_END) + return res; + + if (ret != BZ_OK) + throw Error("error while decompressing bzip2 file"); + } +} + +ref<std::string> compress(const std::string & method, ref<std::string> in) +{ + if (method == "none") + return in; + else if (method == "xz") + return compressXZ(*in); + else if (method == "bzip2") + return compressBzip2(*in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + +ref<std::string> decompress(const std::string & method, ref<std::string> in) +{ + if (method == "none") + return in; + else if (method == "xz") + return decompressXZ(*in); + else if (method == "bzip2") + return decompressBzip2(*in); + else + throw UnknownCompressionMethod(format("unknown compression method ‘%s’") % method); +} + } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index eb1697fc4..33c465df8 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -1,11 +1,16 @@ #pragma once +#include "ref.hh" +#include "types.hh" + #include <string> namespace nix { -std::string compressXZ(const std::string & in); +ref<std::string> compress(const std::string & method, ref<std::string> in); + +ref<std::string> decompress(const std::string & method, ref<std::string> in); -std::string decompressXZ(const std::string & in); +MakeError(UnknownCompressionMethod, Error); } diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh new file mode 100644 index 000000000..47c64deae --- /dev/null +++ b/src/libutil/finally.hh @@ -0,0 +1,12 @@ +#pragma once + +/* A trivial class to run a function at the end of a scope. */ +class Finally +{ +private: + std::function<void()> fun; + +public: + Finally(std::function<void()> fun) : fun(fun) { } + ~Finally() { fun(); } +}; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 647393003..c17f1c4d5 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -33,7 +33,7 @@ Hash::Hash(HashType type) else if (type == htSHA1) hashSize = sha1HashSize; else if (type == htSHA256) hashSize = sha256HashSize; else if (type == htSHA512) hashSize = sha512HashSize; - else throw Error("unknown hash type"); + else abort(); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -64,6 +64,12 @@ bool Hash::operator < (const Hash & h) const } +std::string Hash::to_string(bool base32) const +{ + return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this)); +} + + const string base16Chars = "0123456789abcdef"; @@ -78,15 +84,28 @@ string printHash(const Hash & hash) } +Hash parseHash(const string & s) +{ + string::size_type colon = s.find(':'); + if (colon == string::npos) + throw BadHash(format("invalid hash ‘%s’") % s); + string hts = string(s, 0, colon); + HashType ht = parseHashType(hts); + if (ht == htUnknown) + throw BadHash(format("unknown hash type ‘%s’") % hts); + return parseHash16or32(ht, string(s, colon + 1)); +} + + Hash parseHash(HashType ht, const string & s) { Hash hash(ht); if (s.length() != hash.hashSize * 2) - throw Error(format("invalid hash ‘%1%’") % s); + throw BadHash(format("invalid hash ‘%1%’") % s); for (unsigned int i = 0; i < hash.hashSize; i++) { string s2(s, i * 2, 2); if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw Error(format("invalid hash ‘%1%’") % s); + throw BadHash(format("invalid hash ‘%1%’") % s); std::istringstream str(s2); int n; str >> std::hex >> n; @@ -103,6 +122,7 @@ const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; string printHash32(const Hash & hash) { size_t len = hash.base32Len(); + assert(len); string s; s.reserve(len); @@ -139,7 +159,7 @@ Hash parseHash32(HashType ht, const string & s) for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ if (base32Chars[digit] == c) break; if (digit >= 32) - throw Error(format("invalid base-32 hash ‘%1%’") % s); + throw BadHash(format("invalid base-32 hash ‘%1%’") % s); unsigned int b = n * 5; unsigned int i = b / 8; unsigned int j = b % 8; @@ -161,7 +181,7 @@ Hash parseHash16or32(HashType ht, const string & s) /* base-32 representation */ hash = parseHash32(ht, s); else - throw Error(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") + throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’") % s % printHashType(ht)); return hash; } @@ -322,7 +342,7 @@ string printHashType(HashType ht) else if (ht == htSHA1) return "sha1"; else if (ht == htSHA256) return "sha256"; else if (ht == htSHA512) return "sha512"; - else throw Error("cannot print unknown hash type"); + else abort(); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index bac2ebf2d..02e213fc7 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -7,6 +7,9 @@ namespace nix { +MakeError(BadHash, Error); + + enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; @@ -26,12 +29,15 @@ struct Hash HashType type; - /* Create an unusable hash object. */ + /* Create an unset hash object. */ Hash(); /* Create a zero-filled hash object. */ Hash(HashType type); + /* Check whether a hash is set. */ + operator bool () const { return type != htUnknown; } + /* Check whether two hash are equal. */ bool operator == (const Hash & h2) const; @@ -52,12 +58,16 @@ struct Hash { return (hashSize * 8 - 1) / 5 + 1; } + + std::string to_string(bool base32 = true) const; }; /* Convert a hash to a hexadecimal representation. */ string printHash(const Hash & hash); +Hash parseHash(const string & s); + /* Parse a hexadecimal representation of a hash code. */ Hash parseHash(HashType ht, const string & s); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 4dae33054..98cad00d6 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,6 +6,6 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = -llzma $(OPENSSL_LIBS) +libutil_LDFLAGS = -llzma -lbz2 -pthread $(OPENSSL_LIBS) libutil_LIBS = libformat diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc new file mode 100644 index 000000000..15bb1e175 --- /dev/null +++ b/src/libutil/logging.cc @@ -0,0 +1,79 @@ +#include "logging.hh" +#include "util.hh" + +namespace nix { + +Logger * logger = 0; + +class SimpleLogger : public Logger +{ +public: + + bool systemd, tty; + + SimpleLogger() + { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: c = '3'; break; + case lvlInfo: c = '5'; break; + case lvlTalkative: case lvlChatty: c = '6'; break; + default: c = '7'; + } + prefix = std::string("<") + c + ">"; + } + + writeToStderr(prefix + (tty ? fs.s : filterANSIEscapes(fs.s)) + "\n"); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; + +Verbosity verbosity = lvlInfo; + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % fs.s); + haveWarned = true; + } +} + +void writeToStderr(const string & s) +{ + try { + writeFull(STDERR_FILENO, s); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + +Logger * makeDefaultLogger() +{ + return new SimpleLogger(); +} + +} diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh new file mode 100644 index 000000000..277dff280 --- /dev/null +++ b/src/libutil/logging.hh @@ -0,0 +1,82 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +class Activity; + +class Logger +{ + friend class Activity; + +public: + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void setExpected(const std::string & label, uint64_t value = 1) { } + virtual void setProgress(const std::string & label, uint64_t value = 1) { } + virtual void incExpected(const std::string & label, uint64_t value = 1) { } + virtual void incProgress(const std::string & label, uint64_t value = 1) { } + +private: + + virtual void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) = 0; + + virtual void stopActivity(Activity & activity) = 0; + +}; + +class Activity +{ +public: + Logger & logger; + + Activity(Logger & logger, Verbosity lvl, const FormatOrString & fs) + : logger(logger) + { + logger.startActivity(*this, lvl, fs); + } + + ~Activity() + { + logger.stopActivity(*this); + } +}; + +extern Logger * logger; + +Logger * makeDefaultLogger(); + +extern Verbosity verbosity; /* suppress msgs > this */ + +#define printMsg(level, f) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +} diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 4344d6601..35983aa2c 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -79,6 +79,12 @@ public: { return data.size(); } + + void clear() + { + data.clear(); + lru.clear(); + } }; } diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 349f24f7c..9f5da0915 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -44,6 +44,12 @@ public: } template<typename T2> + ref<T2> cast() + { + return ref<T2>(std::dynamic_pointer_cast<T2>(p)); + } + + template<typename T2> operator ref<T2> () { return ref<T2>((std::shared_ptr<T2>) p); diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index c99c098ac..ebe64ffbd 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -22,11 +22,11 @@ namespace nix { scope. */ -template<class T> +template<class T, class M = std::mutex> class Sync { private: - std::mutex mutex; + M mutex; T data; public: @@ -38,7 +38,7 @@ public: { private: Sync * s; - std::unique_lock<std::mutex> lk; + std::unique_lock<M> lk; friend Sync; Lock(Sync * s) : s(s), lk(s->mutex) { } public: diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc new file mode 100644 index 000000000..32363ecf0 --- /dev/null +++ b/src/libutil/thread-pool.cc @@ -0,0 +1,102 @@ +#include "thread-pool.hh" +#include "affinity.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _maxThreads) + : maxThreads(_maxThreads) +{ + restoreAffinity(); // FIXME + + if (!maxThreads) { + maxThreads = std::thread::hardware_concurrency(); + if (!maxThreads) maxThreads = 1; + } + + debug(format("starting pool of %d threads") % maxThreads); +} + +ThreadPool::~ThreadPool() +{ + std::vector<std::thread> workers; + { + auto state(state_.lock()); + state->quit = true; + std::swap(workers, state->workers); + } + + debug(format("reaping %d worker threads") % workers.size()); + + work.notify_all(); + + for (auto & thr : workers) + thr.join(); +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state(state_.lock()); + assert(!state->quit); + state->left.push(t); + if (state->left.size() > state->workers.size() && state->workers.size() < maxThreads) + state->workers.emplace_back(&ThreadPool::workerEntry, this); + work.notify_one(); +} + +void ThreadPool::process() +{ + while (true) { + auto state(state_.lock()); + if (state->exception) + std::rethrow_exception(state->exception); + if (state->left.empty() && !state->pending) break; + state.wait(done); + } +} + +void ThreadPool::workerEntry() +{ + bool didWork = false; + + while (true) { + work_t w; + { + auto state(state_.lock()); + while (true) { + if (state->quit || state->exception) return; + if (didWork) { + assert(state->pending); + state->pending--; + didWork = false; + } + if (!state->left.empty()) break; + if (!state->pending) + done.notify_all(); + state.wait(work); + } + w = state->left.front(); + state->left.pop(); + state->pending++; + } + + try { + w(); + } catch (std::exception & e) { + auto state(state_.lock()); + if (state->exception) { + if (!dynamic_cast<Interrupted*>(&e)) + printMsg(lvlError, format("error: %s") % e.what()); + } else { + state->exception = std::current_exception(); + work.notify_all(); + done.notify_all(); + } + } + + didWork = true; + } +} + +} + + diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh new file mode 100644 index 000000000..78b63467d --- /dev/null +++ b/src/libutil/thread-pool.hh @@ -0,0 +1,117 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include <queue> +#include <functional> +#include <thread> +#include <map> + +namespace nix { + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t maxThreads = 0); + + ~ThreadPool(); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t maxThreads; + + struct State + { + std::queue<work_t> left; + size_t pending = 0; + std::exception_ptr exception; + std::vector<std::thread> workers; + bool quit = false; + }; + + Sync<State> state_; + + std::condition_variable work, done; + + void workerEntry(); +}; + +/* Process in parallel a set of items of type T that have a partial + ordering between them. Thus, any item is only processed after all + its dependencies have been processed. */ +template<typename T> +void processGraph( + ThreadPool & pool, + const std::set<T> & nodes, + std::function<std::set<T>(const T &)> getEdges, + std::function<void(const T &)> processNode) +{ + struct Graph { + std::set<T> left; + std::map<T, std::set<T>> refs, rrefs; + std::function<void(T)> wrap; + }; + + ref<Sync<Graph>> graph_ = make_ref<Sync<Graph>>(); + + auto wrapWork = [&pool, graph_, processNode](const T & node) { + processNode(node); + + /* Enqueue work for all nodes that were waiting on this one. */ + { + auto graph(graph_->lock()); + graph->left.erase(node); + for (auto & rref : graph->rrefs[node]) { + auto & refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) + pool.enqueue(std::bind(graph->wrap, rref)); + } + } + }; + + { + auto graph(graph_->lock()); + graph->left = nodes; + graph->wrap = wrapWork; + } + + /* Build the dependency graph; enqueue all nodes with no + dependencies. */ + for (auto & node : nodes) { + auto refs = getEdges(node); + { + auto graph(graph_->lock()); + for (auto & ref : refs) + if (ref != node && graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + pool.enqueue(std::bind(graph->wrap, node)); + } + } +} + +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 33aaf5fc9..bd192b850 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -89,14 +89,4 @@ typedef list<Path> Paths; typedef set<Path> PathSet; -typedef enum { - lvlError = 0, - lvlInfo, - lvlTalkative, - lvlChatty, - lvlDebug, - lvlVomit -} Verbosity; - - } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 25246a3e8..67558cc0b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -356,8 +356,7 @@ void deletePath(const Path & path) void deletePath(const Path & path, unsigned long long & bytesFreed) { - startNest(nest, lvlDebug, - format("recursively deleting path ‘%1%’") % path); + Activity act(*logger, lvlDebug, format("recursively deleting path ‘%1%’") % path); bytesFreed = 0; _deletePath(path, bytesFreed); } @@ -403,6 +402,18 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +Path getCacheDir() +{ + Path cacheDir = getEnv("XDG_CACHE_HOME"); + if (cacheDir.empty()) { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set"); + cacheDir = homeDir + "/.cache"; + } + return cacheDir; +} + + Paths createDirs(const Path & path) { Paths created; @@ -444,113 +455,6 @@ void replaceSymlink(const Path & target, const Path & link) } -LogType logType = ltPretty; -Verbosity verbosity = lvlInfo; - -static int nestingLevel = 0; - - -Nest::Nest() -{ - nest = false; -} - - -Nest::~Nest() -{ - close(); -} - - -static string escVerbosity(Verbosity level) -{ - return std::to_string((int) level); -} - - -void Nest::open(Verbosity level, const FormatOrString & fs) -{ - if (level <= verbosity) { - if (logType == ltEscapes) - std::cerr << "\033[" << escVerbosity(level) << "p" - << fs.s << "\n"; - else - printMsg_(level, fs); - nest = true; - nestingLevel++; - } -} - - -void Nest::close() -{ - if (nest) { - nestingLevel--; - if (logType == ltEscapes) - std::cerr << "\033[q"; - nest = false; - } -} - - -void printMsg_(Verbosity level, const FormatOrString & fs) -{ - checkInterrupt(); - if (level > verbosity) return; - - string prefix; - if (logType == ltPretty) - for (int i = 0; i < nestingLevel; i++) - prefix += "| "; - else if (logType == ltEscapes && level != lvlInfo) - prefix = "\033[" + escVerbosity(level) + "s"; - else if (logType == ltSystemd) { - char c; - switch (level) { - case lvlError: c = '3'; break; - case lvlInfo: c = '5'; break; - case lvlTalkative: case lvlChatty: c = '6'; break; - default: c = '7'; - } - prefix = string("<") + c + ">"; - } - - string s = (format("%1%%2%\n") % prefix % fs.s).str(); - if (!isatty(STDERR_FILENO)) s = filterANSIEscapes(s); - writeToStderr(s); -} - - -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - printMsg(lvlError, format("warning: %1%") % fs.s); - haveWarned = true; - } -} - - -void writeToStderr(const string & s) -{ - try { - if (_writeToStderr) - _writeToStderr((const unsigned char *) s.data(), s.size()); - else - writeFull(STDERR_FILENO, s); - } catch (SysError & e) { - /* Ignore failing writes to stderr if we're in an exception - handler, otherwise throw an exception. We need to ignore - write errors in exception handlers to ensure that cleanup - code runs to completion if the other side of stderr has - been closed unexpectedly. */ - if (!std::uncaught_exception()) throw; - } -} - - -void (*_writeToStderr) (const unsigned char * buf, size_t count) = 0; - - void readFull(int fd, unsigned char * buf, size_t count) { while (count) { @@ -941,7 +845,8 @@ static pid_t doFork(bool allowVfork, std::function<void()> fun) pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { auto wrapper = [&]() { - if (!options.allowVfork) _writeToStderr = 0; + if (!options.allowVfork) + logger = makeDefaultLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) @@ -1062,13 +967,15 @@ void restoreSIGPIPE() volatile sig_atomic_t _isInterrupted = 0; +thread_local bool interruptThrown = false; + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!std::uncaught_exception()) { - _isInterrupted = 0; + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; throw Interrupted("interrupted by the user"); } } @@ -1175,6 +1082,12 @@ bool statusOk(int status) } +bool hasPrefix(const string & s, const string & suffix) +{ + return s.compare(0, suffix.size(), suffix) == 0; +} + + bool hasSuffix(const string & s, const string & suffix) { return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3606f6ec9..f3f0f92a0 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,6 +1,7 @@ #pragma once #include "types.hh" +#include "logging.hh" #include <sys/types.h> #include <sys/stat.h> @@ -102,6 +103,9 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +/* Return the path to $XDG_CACHE_HOME/.cache. */ +Path getCacheDir(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); @@ -122,54 +126,6 @@ T singleton(const A & a) } -/* Messages. */ - - -typedef enum { - ltPretty, /* nice, nested output */ - ltEscapes, /* nesting indicated using escape codes (for log2xml) */ - ltFlat, /* no nesting */ - ltSystemd, /* use systemd severity prefixes */ -} LogType; - -extern LogType logType; -extern Verbosity verbosity; /* suppress msgs > this */ - -class Nest -{ -private: - bool nest; -public: - Nest(); - ~Nest(); - void open(Verbosity level, const FormatOrString & fs); - void close(); -}; - -void printMsg_(Verbosity level, const FormatOrString & fs); - -#define startNest(varName, level, f) \ - Nest varName; \ - if (level <= verbosity) { \ - varName.open(level, (f)); \ - } - -#define printMsg(level, f) \ - do { \ - if (level <= nix::verbosity) { \ - nix::printMsg_(level, (f)); \ - } \ - } while (0) - -#define debug(f) printMsg(lvlDebug, f) - -void warnOnce(bool & haveWarned, const FormatOrString & fs); - -void writeToStderr(const string & s); - -extern void (*_writeToStderr) (const unsigned char * buf, size_t count); - - /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ void readFull(int fd, unsigned char * buf, size_t count); @@ -316,6 +272,8 @@ void restoreSIGPIPE(); extern volatile sig_atomic_t _isInterrupted; +extern thread_local bool interruptThrown; + void _interrupted(); void inline checkInterrupt() @@ -375,6 +333,10 @@ template<class N> bool string2Float(const string & s, N & n) } +/* Return true iff `s' starts with `prefix'. */ +bool hasPrefix(const string & s, const string & prefix); + + /* Return true iff `s' ends in `suffix'. */ bool hasSuffix(const string & s, const string & suffix); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 5189c9b4c..3c2e05210 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -33,29 +33,43 @@ using namespace nix; static FdSource from(STDIN_FILENO); static FdSink to(STDOUT_FILENO); -bool canSendStderr; +static bool canSendStderr; +static Logger * defaultLogger; -/* This function is called anytime we want to write something to - stderr. If we're in a state where the protocol allows it (i.e., - when canSendStderr), send the message to the client over the - socket. */ -static void tunnelStderr(const unsigned char * buf, size_t count) + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +class TunnelLogger : public Logger { - if (canSendStderr) { - try { - to << STDERR_NEXT; - writeString(buf, count, to); - to.flush(); - } catch (...) { - /* Write failed; that means that the other side is - gone. */ - canSendStderr = false; - throw; - } - } else - writeFull(STDERR_FILENO, buf, count); -} + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + if (canSendStderr) { + try { + to << STDERR_NEXT << (fs.s + "\n"); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + canSendStderr = false; + throw; + } + } else + defaultLogger->log(lvl, fs); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + log(lvl, fs); + } + + void stopActivity(Activity & activity) override + { + } +}; /* startWork() means that we're starting an operation for which we @@ -199,7 +213,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe case wopQueryPathHash: { Path path = readStorePath(from); startWork(); - Hash hash = store->queryPathHash(path); + auto hash = store->queryPathInfo(path)->narHash; stopWork(); to << printHash(hash); break; @@ -213,7 +227,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe startWork(); PathSet paths; if (op == wopQueryReferences) - store->queryReferences(path, paths); + paths = store->queryPathInfo(path)->references; else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) @@ -237,7 +251,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe case wopQueryDeriver: { Path path = readStorePath(from); startWork(); - Path deriver = store->queryDeriver(path); + auto deriver = store->queryPathInfo(path)->deriver; stopWork(); to << deriver; break; @@ -429,9 +443,9 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe if (GET_PROTOCOL_MINOR(clientVersion) >= 2) settings.useBuildHook = readInt(from) != 0; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { - settings.buildVerbosity = (Verbosity) readInt(from); - logType = (LogType) readInt(from); - settings.printBuildTrace = readInt(from) != 0; + settings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace } if (GET_PROTOCOL_MINOR(clientVersion) >= 6) settings.set("build-cores", std::to_string(readInt(from))); @@ -493,30 +507,25 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe break; } - case wopQueryFailedPaths: { - startWork(); - PathSet paths = store->queryFailedPaths(); - stopWork(); - to << paths; - break; - } - - case wopClearFailedPaths: { - PathSet paths = readStrings<PathSet>(from); - startWork(); - store->clearFailedPaths(paths); - stopWork(); - to << 1; - break; - } - case wopQueryPathInfo: { Path path = readStorePath(from); + std::shared_ptr<const ValidPathInfo> info; startWork(); - ValidPathInfo info = store->queryPathInfo(path); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath &) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } stopWork(); - to << info.deriver << printHash(info.narHash) << info.references - << info.registrationTime << info.narSize; + if (info) { + to << 1 << info->deriver << printHash(info->narHash) << info->references + << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs; + } + } else + to << 0; break; } @@ -539,6 +548,18 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe break; } + case wopAddSignatures: { + Path path = readStorePath(from); + StringSet sigs = readStrings<StringSet>(from); + startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + stopWork(); + to << 1; + break; + } + default: throw Error(format("invalid operation %1%") % op); } @@ -550,7 +571,8 @@ static void processConnection(bool trusted) MonitorFdHup monitor(from.fd); canSendStderr = false; - _writeToStderr = tunnelStderr; + defaultLogger = logger; + logger = new TunnelLogger(); /* Exchange the greeting. */ unsigned int magic = readInt(from); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a9d1ed024..6bc8d79bc 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -996,7 +996,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) try { if (i.hasFailed()) continue; - startNest(nest, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); + Activity act(*logger, lvlDebug, format("outputting query result ‘%1%’") % i.attrPath); if (globals.prebuiltOnly && validPaths.find(i.queryOutPath()) == validPaths.end() && diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 4e0e28c11..e378d8196 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -63,8 +63,8 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, if (drvPath != "") mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); - // Copy each output. - DrvInfo::Outputs outputs = i.queryOutputs(); + // Copy each output meant for installation. + DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 81c1c8d56..7dce08400 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -19,7 +19,7 @@ using namespace nix; static Expr * parseStdin(EvalState & state) { - startNest(nest, lvlTalkative, format("parsing standard input")); + Activity act(*logger, lvlTalkative, format("parsing standard input")); return state.parseExprFromString(drainFD(0), absPath(".")); } diff --git a/src/nix-log2xml/local.mk b/src/nix-log2xml/local.mk deleted file mode 100644 index 09c848c17..000000000 --- a/src/nix-log2xml/local.mk +++ /dev/null @@ -1,5 +0,0 @@ -programs += nix-log2xml - -nix-log2xml_DIR := $(d) - -nix-log2xml_SOURCES := $(d)/log2xml.cc diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc deleted file mode 100644 index 31cea60c3..000000000 --- a/src/nix-log2xml/log2xml.cc +++ /dev/null @@ -1,201 +0,0 @@ -#include <vector> -#include <iostream> -#include <cstdio> -#include <string> -#include <cstring> - -using namespace std; - - -struct Decoder -{ - enum { stTop, stEscape, stCSI } state; - string line; - bool inHeader; - int level; - vector<int> args; - bool newNumber; - int priority; - bool ignoreLF; - int lineNo, charNo; - bool warning; - bool error; - - Decoder() - { - state = stTop; - line = ""; - inHeader = false; - level = 0; - priority = 1; - ignoreLF = false; - lineNo = 1; - charNo = 0; - warning = false; - error = false; - } - - void pushChar(char c); - - void finishLine(); - - void decodeFile(istream & st); -}; - - -void Decoder::pushChar(char c) -{ - if (c == '\n') { - lineNo++; - charNo = 0; - } else charNo++; - - switch (state) { - - case stTop: - if (c == '\e') { - state = stEscape; - } else if (c == '\n' && !ignoreLF) { - finishLine(); - } else line += c; - break; - - case stEscape: - if (c == '[') { - state = stCSI; - args.clear(); - newNumber = true; - } else - state = stTop; /* !!! wrong */ - break; - - case stCSI: - if (c >= 0x40 && c != 0x7e) { - state = stTop; - switch (c) { - case 'p': - if (line.size()) finishLine(); - level++; - inHeader = true; - cout << "<nest>" << endl; - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'q': - if (line.size()) finishLine(); - if (level > 0) { - level--; - cout << "</nest>" << endl; - } else - cerr << "not enough nesting levels at line " - << lineNo << ", character " << charNo << endl; - break; - case 's': - if (line.size()) finishLine(); - priority = args.size() >= 1 ? args[0] : 1; - break; - case 'a': - ignoreLF = true; - break; - case 'b': - ignoreLF = false; - break; - case 'e': - error = true; - break; - case 'w': - warning = true; - break; - } - } else if (c >= '0' && c <= '9') { - int n = 0; - if (!newNumber) { - n = args.back() * 10; - args.pop_back(); - } - n += c - '0'; - args.push_back(n); - } - break; - - } -} - - -void Decoder::finishLine() -{ - string storeDir = "/nix/store/"; - int sz = storeDir.size(); - string tag = inHeader ? "head" : "line"; - cout << "<" << tag; - if (priority != 1) cout << " priority='" << priority << "'"; - if (warning) cout << " warning='1'"; - if (error) cout << " error='1'"; - cout << ">"; - - for (unsigned int i = 0; i < line.size(); i++) { - - if (line[i] == '<') cout << "<"; - else if (line[i] == '&') cout << "&"; - else if (line[i] == '\r') ; /* ignore carriage return */ - else if (line[i] == '\n') cout << "\n"; - else if (line[i] >= 0 && line[i] < 32 && line[i] != 9) cout << "�"; - else if (i + sz + 33 < line.size() && - string(line, i, sz) == storeDir && - line[i + sz + 32] == '-') - { - int j = i + sz + 32; - /* skip name */ - while (!strchr("/\n\r\t ()[]:;?<>", line[j])) j++; - int k = j; - while (!strchr("\n\r\t ()[]:;?<>", line[k])) k++; - // !!! escaping - cout << "<storeref>" - << "<storedir>" - << string(line, i, sz) - << "</storedir>" - << "<hash>" - << string(line, i + sz, 32) - << "</hash>" - << "<name>" - << string(line, i + sz + 32, j - (i + sz + 32)) - << "</name>" - << "<path>" - << string(line, j, k - j) - << "</path>" - << "</storeref>"; - i = k - 1; - } else cout << line[i]; - } - - cout << "</" << tag << ">" << endl; - line = ""; - inHeader = false; - priority = 1; - warning = false; - error = false; -} - - -void Decoder::decodeFile(istream & st) -{ - int c; - - cout << "<logfile>" << endl; - - while ((c = st.get()) != EOF) { - pushChar(c); - } - - if (line.size()) finishLine(); - - while (level--) cout << "</nest>" << endl; - - cout << "</logfile>" << endl; -} - - -int main(int argc, char * * argv) -{ - Decoder dec; - dec.decodeFile(cin); -} diff --git a/src/nix-log2xml/logfile.css b/src/nix-log2xml/logfile.css deleted file mode 100644 index ed390d64a..000000000 --- a/src/nix-log2xml/logfile.css +++ /dev/null @@ -1,86 +0,0 @@ -body { - font-family: sans-serif; - background: white; -} - - -ul.nesting, ul.toplevel { - padding: 0; - margin: 0; -} - -ul.toplevel { - list-style-type: none; -} - -ul.nesting li.line, ul.nesting li.lastline { - position: relative; - list-style-type: none; -} - -ul.nesting li.line { - padding-left: 1.1em; -} - -ul.nesting li.lastline { - padding-left: 1.2em; // for the 0.1em border-left in .lastline > .lineconn -} - -li.line { - border-left: 0.1em solid #6185a0; -} - -li.line > span.lineconn, li.lastline > span.lineconn { - position: absolute; - height: 0.65em; - left: 0em; - width: 1em; - border-bottom: 0.1em solid #6185a0; -} - -li.lastline > span.lineconn { - border-left: 0.1em solid #6185a0; -} - - -em.storeref { - color: #500000; - position: relative; - width: 100%; -} - -em.storeref:hover { - background-color: #eeeeee; -} - -*.popup { - display: none; -/* background: url('http://losser.st-lab.cs.uu.nl/~mbravenb/menuback.png') repeat; */ - background: #ffffcd; - border: solid #555555 1px; - position: absolute; - top: 0em; - left: 0em; - margin: 0; - padding: 0; - z-index: 100; -} - -em.storeref:hover span.popup { - display: inline; -} - - -.toggle { - text-decoration: none; -} - -.showTree, .hideTree { - font-family: monospace; - font-size: larger; -} - -.error { - color: #ff0000; - font-weight: bold; -}
\ No newline at end of file diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index c65961a15..64da10513 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -162,7 +162,7 @@ int main(int argc, char * * argv) AutoDelete tmpDir(createTempDir(), true); Path tmpFile = (Path) tmpDir + "/tmp"; - writeFile(tmpFile, result.data); + writeFile(tmpFile, *result.data); /* Optionally unpack the file. */ if (unpack) { @@ -186,7 +186,7 @@ int main(int argc, char * * argv) /* FIXME: inefficient; addToStore() will also hash this. */ - hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, result.data); + hash = unpack ? hashPath(ht, tmpFile).first : hashString(ht, *result.data); if (expectedHash != Hash(ht) && expectedHash != hash) throw Error(format("hash mismatch for ‘%1%’") % uri); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 8735cf9b6..356a82510 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -110,19 +110,13 @@ void printDotGraph(ref<Store> store, const PathSet & roots) cout << makeNode(path, symbolicName(path), "#ff0000"); - PathSet references; - store->queryReferences(path, references); - - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) - { - if (*i != path) { - workList.insert(*i); - cout << makeEdge(*i, path); + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); } } - #if 0 StoreExpr ne = storeExprFromPath(path); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index a258c77c3..653a95f21 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,6 +9,7 @@ #include "util.hh" #include "worker-protocol.hh" #include "xmlgraph.hh" +#include "compression.hh" #include <iostream> #include <algorithm> @@ -51,7 +52,7 @@ ref<LocalStore> ensureLocalStore() static Path useDeriver(Path path) { if (isDerivation(path)) return path; - Path drvPath = store->queryDeriver(path); + Path drvPath = store->queryPathInfo(path)->deriver; if (drvPath == "") throw Error(format("deriver of path ‘%1%’ is not known") % path); return drvPath; @@ -247,8 +248,7 @@ static void printTree(const Path & path, cout << format("%1%%2%\n") % firstPad % path; - PathSet references; - store->queryReferences(path, references); + auto references = store->queryPathInfo(path)->references; /* Topologically sort under the relation A < B iff A \in closure(B). That is, if derivation A is an (possibly indirect) @@ -335,7 +335,10 @@ static void opQuery(Strings opFlags, Strings opArgs) PathSet ps = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); - else if (query == qReferences) store->queryReferences(j, paths); + else if (query == qReferences) { + for (auto & p : store->queryPathInfo(j)->references) + paths.insert(p); + } else if (query == qReferrers) store->queryReferrers(j, paths); else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); } @@ -349,7 +352,7 @@ static void opQuery(Strings opFlags, Strings opArgs) case qDeriver: for (auto & i : opArgs) { - Path deriver = store->queryDeriver(followLinksToStorePath(i)); + Path deriver = store->queryPathInfo(followLinksToStorePath(i))->deriver; cout << format("%1%\n") % (deriver == "" ? "unknown-deriver" : deriver); } @@ -372,12 +375,12 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & i : opArgs) { PathSet paths = maybeUseOutputs(followLinksToStorePath(i), useOutput, forceRealise); for (auto & j : paths) { - ValidPathInfo info = store->queryPathInfo(j); + auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info.narHash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(info.narHash); + assert(info->narHash.type == htSHA256); + cout << format("sha256:%1%\n") % printHash32(info->narHash); } else if (query == qSize) - cout << format("%1%\n") % info.narSize; + cout << format("%1%\n") % info->narSize; } } break; @@ -500,21 +503,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) } else if (pathExists(logBz2Path)) { - AutoCloseFD fd = open(logBz2Path.c_str(), O_RDONLY); - FILE * f = 0; - if (fd == -1 || (f = fdopen(fd.borrow(), "r")) == 0) - throw SysError(format("opening file ‘%1%’") % logBz2Path); - int err; - BZFILE * bz = BZ2_bzReadOpen(&err, f, 0, 0, 0, 0); - if (!bz) throw Error(format("cannot open bzip2 file ‘%1%’") % logBz2Path); - unsigned char buf[128 * 1024]; - do { - int n = BZ2_bzRead(&err, bz, buf, sizeof(buf)); - if (err != BZ_OK && err != BZ_STREAM_END) - throw Error(format("error reading bzip2 file ‘%1%’") % logBz2Path); - writeFull(STDOUT_FILENO, buf, n); - } while (err != BZ_STREAM_END); - BZ2_bzReadClose(&err, bz); + std::cout << *decompress("bzip2", make_ref<std::string>(readFile(logBz2Path))); found = true; break; } @@ -782,12 +771,14 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) for (auto & i : opArgs) { Path path = followLinksToStorePath(i); printMsg(lvlTalkative, format("checking path ‘%1%’...") % path); - ValidPathInfo info = store->queryPathInfo(path); - HashResult current = hashPath(info.narHash.type, path); - if (current.first != info.narHash) { + auto info = store->queryPathInfo(path); + HashSink sink(info->narHash.type); + store->narFromPath(path, sink); + auto current = sink.finish(); + if (current.first != info->narHash) { printMsg(lvlError, format("path ‘%1%’ was modified! expected hash ‘%2%’, got ‘%3%’") - % path % printHash(info.narHash) % printHash(current.first)); + % path % printHash(info->narHash) % printHash(current.first)); status = 1; } } @@ -819,24 +810,6 @@ static void opOptimise(Strings opFlags, Strings opArgs) store->optimiseStore(); } -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments expected"); - PathSet failed = store->queryFailedPaths(); - for (auto & i : failed) - cout << format("%1%\n") % i; -} - - -static void opClearFailedPaths(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) - throw UsageError("no flags expected"); - store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); -} - - /* Serve the nix store in a way usable by a restricted ssh user. */ static void opServe(Strings opFlags, Strings opArgs) { @@ -917,13 +890,14 @@ static void opServe(Strings opFlags, Strings opArgs) PathSet paths = readStorePaths<PathSet>(in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { - if (!store->isValidPath(i)) - continue; - ValidPathInfo info = store->queryPathInfo(i); - out << info.path << info.deriver << info.references; - // !!! Maybe we want compression? - out << info.narSize // downloadSize - << info.narSize; + try { + auto info = store->queryPathInfo(i); + out << info->path << info->deriver << info->references; + // !!! Maybe we want compression? + out << info->narSize // downloadSize + << info->narSize; + } catch (InvalidPath &) { + } } out << ""; break; @@ -1100,10 +1074,6 @@ int main(int argc, char * * argv) op = opRepairPath; else if (*arg == "--optimise" || *arg == "--optimize") op = opOptimise; - else if (*arg == "--query-failed-paths") - op = opQueryFailedPaths; - else if (*arg == "--clear-failed-paths") - op = opClearFailedPaths; else if (*arg == "--serve") op = opServe; else if (*arg == "--generate-binary-cache-key") diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc index b6e1c1c4b..0f7be7f7a 100644 --- a/src/nix-store/xmlgraph.cc +++ b/src/nix-store/xmlgraph.cc @@ -50,15 +50,10 @@ void printXmlGraph(ref<Store> store, const PathSet & roots) cout << makeNode(path); - PathSet references; - store->queryReferences(path, references); - - for (PathSet::iterator i = references.begin(); - i != references.end(); ++i) - { - if (*i != path) { - workList.insert(*i); - cout << makeEdge(*i, path); + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); } } diff --git a/src/nix/command.cc b/src/nix/command.cc index 9c80f4309..c8d91737d 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -5,6 +5,21 @@ namespace nix { Commands * RegisterCommand::commands = 0; +void Command::printHelp(const string & programName, std::ostream & out) +{ + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto & ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } +} + MultiCommand::MultiCommand(const Commands & _commands) : commands(_commands) { @@ -57,9 +72,47 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } +StoreCommand::StoreCommand() +{ + storeUri = getEnv("NIX_REMOTE"); + + mkFlag(0, "store", "store-uri", "URI of the Nix store to use", &storeUri); +} + void StoreCommand::run() { - run(openStore()); + run(openStoreAt(storeUri)); +} + +StorePathsCommand::StorePathsCommand() +{ + expectArgs("paths", &storePaths); + mkFlag('r', "recursive", "apply operation to closure of the specified paths", &recursive); + mkFlag(0, "all", "apply operation to the entire store", &all); +} + +void StorePathsCommand::run(ref<Store> store) +{ + if (all) { + if (storePaths.size()) + throw UsageError("‘--all’ does not expect arguments"); + for (auto & p : store->queryAllValidPaths()) + storePaths.push_back(p); + } + + else { + for (auto & storePath : storePaths) + storePath = followLinksToStorePath(storePath); + + if (recursive) { + PathSet closure; + for (auto & storePath : storePaths) + store->computeFSClosure(storePath, closure, false, false); + storePaths = store->topoSortPaths(closure); + } + } + + run(store, storePaths); } } diff --git a/src/nix/command.hh b/src/nix/command.hh index 27c3ab7f2..34affc43d 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -11,6 +11,18 @@ struct Command : virtual Args virtual std::string name() = 0; virtual void prepare() { }; virtual void run() = 0; + + struct Example + { + std::string description; + std::string command; + }; + + typedef std::list<Example> Examples; + + virtual Examples examples() { return Examples(); } + + void printHelp(const string & programName, std::ostream & out) override; }; class Store; @@ -18,13 +30,30 @@ class Store; /* A command that require a Nix store. */ struct StoreCommand : virtual Command { - bool reserveSpace; - StoreCommand(bool reserveSpace = true) - : reserveSpace(reserveSpace) { }; + std::string storeUri; + StoreCommand(); void run() override; virtual void run(ref<Store>) = 0; }; +/* A command that operates on zero or more store paths. */ +struct StorePathsCommand : public StoreCommand +{ +private: + + Paths storePaths; + bool recursive = false; + bool all = false; + +public: + + StorePathsCommand(); + + virtual void run(ref<Store> store, Paths storePaths) = 0; + + void run(ref<Store> store) override; +}; + typedef std::map<std::string, ref<Command>> Commands; /* An argument parser that supports multiple subcommands, diff --git a/src/nix/copy.cc b/src/nix/copy.cc new file mode 100644 index 000000000..be51fee62 --- /dev/null +++ b/src/nix/copy.cc @@ -0,0 +1,83 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdCopy : StorePathsCommand +{ + std::string srcUri, dstUri; + + CmdCopy() + { + mkFlag(0, "from", "store-uri", "URI of the source Nix store", &srcUri); + mkFlag(0, "to", "store-uri", "URI of the destination Nix store", &dstUri); + } + + std::string name() override + { + return "copy"; + } + + std::string description() override + { + return "copy paths between Nix stores"; + } + + Examples examples() override + { + return { + Example{ + "To copy Firefox to the local store to a binary cache in file:///tmp/cache:", + "nix copy --to file:///tmp/cache -r $(type -p firefox)" + }, + }; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (srcUri.empty() && dstUri.empty()) + throw UsageError("you must pass ‘--from’ and/or ‘--to’"); + + ref<Store> srcStore = srcUri.empty() ? store : openStoreAt(srcUri); + ref<Store> dstStore = dstUri.empty() ? store : openStoreAt(dstUri); + + std::string copiedLabel = "copied"; + + logger->setExpected(copiedLabel, storePaths.size()); + + ThreadPool pool; + + processGraph<Path>(pool, + PathSet(storePaths.begin(), storePaths.end()), + + [&](const Path & storePath) { + return srcStore->queryPathInfo(storePath)->references; + }, + + [&](const Path & storePath) { + checkInterrupt(); + + if (!dstStore->isValidPath(storePath)) { + Activity act(*logger, lvlInfo, format("copying ‘%s’...") % storePath); + + StringSink sink; + srcStore->exportPaths({storePath}, false, sink); + + StringSource source(*sink.s); + dstStore->importPaths(false, source, 0); + + logger->incProgress(copiedLabel); + } else + logger->incExpected(copiedLabel, -1); + }); + + pool.process(); + } +}; + +static RegisterCommand r1(make_ref<CmdCopy>()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 2005ec5f9..440ced97d 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -7,6 +7,7 @@ #include "legacy.hh" #include "shared.hh" #include "store-api.hh" +#include "progress-bar.hh" namespace nix { @@ -26,6 +27,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { + settings.verboseBuild = false; + initNix(); initGC(); @@ -42,6 +45,8 @@ void mainWrapped(int argc, char * * argv) assert(args.command); + StartProgressBar bar; + args.command->prepare(); args.command->run(); } diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc new file mode 100644 index 000000000..c61fe7ff1 --- /dev/null +++ b/src/nix/path-info.cc @@ -0,0 +1,85 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +#include <iomanip> +#include <algorithm> + +using namespace nix; + +struct CmdPathInfo : StorePathsCommand +{ + bool showSize = false; + bool showClosureSize = false; + bool showSigs = false; + + CmdPathInfo() + { + mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); + mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); + mkFlag(0, "sigs", "show signatures", &showSigs); + } + + std::string name() override + { + return "path-info"; + } + + std::string description() override + { + return "query information about store paths"; + } + + Examples examples() override + { + return { + Example{ + "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", + "nix path-info -rS /run/current-system | sort -nk2" + }, + Example{ + "To check the existence of a path in a binary cache:", + "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" + }, + }; + } + + void run(ref<Store> store, Paths storePaths) override + { + size_t pathLen = 0; + for (auto & storePath : storePaths) + pathLen = std::max(pathLen, storePath.size()); + + for (auto storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + storePath = info->path; // FIXME: screws up padding + + std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + + if (showSize) { + std::cout << '\t' << std::setw(11) << info->narSize; + } + + if (showClosureSize) { + size_t totalSize = 0; + PathSet closure; + store->computeFSClosure(storePath, closure, false, false); + for (auto & p : closure) + totalSize += store->queryPathInfo(p)->narSize; + std::cout << '\t' << std::setw(11) << totalSize; + } + + if (showSigs) { + std::cout << '\t'; + Strings ss; + if (info->ultimate) ss.push_back("ultimate"); + for (auto & sig : info->sigs) ss.push_back(sig); + std::cout << concatStringsSep(" ", ss); + } + + std::cout << std::endl; + } + } +}; + +static RegisterCommand r1(make_ref<CmdPathInfo>()); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc new file mode 100644 index 000000000..659d6572a --- /dev/null +++ b/src/nix/progress-bar.cc @@ -0,0 +1,157 @@ +#include "progress-bar.hh" +#include "util.hh" +#include "sync.hh" + +#include <map> + +namespace nix { + +class ProgressBar : public Logger +{ +private: + + struct ActInfo + { + Activity * activity; + Verbosity lvl; + std::string s; + }; + + struct Progress + { + uint64_t expected = 0, progress = 0; + }; + + struct State + { + std::list<ActInfo> activities; + std::map<Activity *, std::list<ActInfo>::iterator> its; + std::map<std::string, Progress> progress; + }; + + Sync<State> state_; + +public: + + ~ProgressBar() + { + auto state(state_.lock()); + assert(state->activities.empty()); + writeToStderr("\r\e[K"); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + auto state(state_.lock()); + log(*state, lvl, fs.s); + } + + void log(State & state, Verbosity lvl, const std::string & s) + { + writeToStderr("\r\e[K" + s + "\n"); + update(state); + } + + void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + auto state(state_.lock()); + state->activities.emplace_back(ActInfo{&activity, lvl, fs.s}); + state->its.emplace(&activity, std::prev(state->activities.end())); + update(*state); + } + + void stopActivity(Activity & activity) override + { + auto state(state_.lock()); + auto i = state->its.find(&activity); + if (i == state->its.end()) return; + state->activities.erase(i->second); + state->its.erase(i); + update(*state); + } + + void setExpected(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].expected = value; + } + + void setProgress(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].progress = value; + } + + void incExpected(const std::string & label, uint64_t value) override + { + auto state(state_.lock()); + state->progress[label].expected += value; + } + + void incProgress(const std::string & label, uint64_t value) + { + auto state(state_.lock()); + state->progress[label].progress += value; + } + + void update() + { + auto state(state_.lock()); + } + + void update(State & state) + { + std::string line = "\r"; + + std::string status = getStatus(state); + if (!status.empty()) { + line += '['; + line += status; + line += "]"; + } + + if (!state.activities.empty()) { + if (!status.empty()) line += " "; + line += state.activities.rbegin()->s; + } + + line += "\e[K"; + writeToStderr(line); + } + + std::string getStatus(State & state) + { + std::string res; + for (auto & p : state.progress) + if (p.second.expected || p.second.progress) { + if (!res.empty()) res += ", "; + res += std::to_string(p.second.progress); + if (p.second.expected) { + res += "/"; + res += std::to_string(p.second.expected); + } + res += " "; res += p.first; + } + return res; + } +}; + +StartProgressBar::StartProgressBar() +{ + if (isatty(STDERR_FILENO)) { + prev = logger; + logger = new ProgressBar(); + } +} + +StartProgressBar::~StartProgressBar() +{ + if (prev) { + auto bar = logger; + logger = prev; + delete bar; + } +} + +} diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh new file mode 100644 index 000000000..d2e44f7c4 --- /dev/null +++ b/src/nix/progress-bar.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "logging.hh" + +namespace nix { + +class StartProgressBar +{ + Logger * prev = 0; +public: + StartProgressBar(); + ~StartProgressBar(); +}; + +} diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc new file mode 100644 index 000000000..9932aa4a9 --- /dev/null +++ b/src/nix/sigs.cc @@ -0,0 +1,139 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdCopySigs : StorePathsCommand +{ + Strings substituterUris; + + CmdCopySigs() + { + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + } + + std::string name() override + { + return "copy-sigs"; + } + + std::string description() override + { + return "copy path signatures from substituters (like binary caches)"; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (substituterUris.empty()) + throw UsageError("you must specify at least one substituter using ‘-s’"); + + // FIXME: factor out commonality with MixVerify. + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + ThreadPool pool; + + std::string doneLabel = "done"; + std::atomic<size_t> added{0}; + + logger->setExpected(doneLabel, storePaths.size()); + + auto doPath = [&](const Path & storePath) { + Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath); + + checkInterrupt(); + + auto info = store->queryPathInfo(storePath); + + StringSet newSigs; + + for (auto & store2 : substituters) { + try { + auto info2 = store2->queryPathInfo(storePath); + + /* Don't import signatures that don't match this + binary. */ + if (info->narHash != info2->narHash || + info->narSize != info2->narSize || + info->references != info2->references) + continue; + + for (auto & sig : info2->sigs) + if (!info->sigs.count(sig)) + newSigs.insert(sig); + } catch (InvalidPath &) { + } + } + + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } + + logger->incProgress(doneLabel); + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + printMsg(lvlInfo, format("imported %d signatures") % added); + } +}; + +static RegisterCommand r1(make_ref<CmdCopySigs>()); + +struct CmdSignPaths : StorePathsCommand +{ + Path secretKeyFile; + + CmdSignPaths() + { + mkFlag('k', "key-file", {"file"}, "file containing the secret signing key", &secretKeyFile); + } + + std::string name() override + { + return "sign-paths"; + } + + std::string description() override + { + return "sign the specified paths"; + } + + void run(ref<Store> store, Paths storePaths) override + { + if (secretKeyFile.empty()) + throw UsageError("you must specify a secret key file using ‘-k’"); + + SecretKey secretKey(readFile(secretKeyFile)); + + size_t added{0}; + + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + + auto info2(*info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); + + if (!info->sigs.count(*info2.sigs.begin())) { + store->addSignatures(storePath, info2.sigs); + added++; + } + } + + printMsg(lvlInfo, format("added %d signatures") % added); + } +}; + +static RegisterCommand r3(make_ref<CmdSignPaths>()); diff --git a/src/nix/verify.cc b/src/nix/verify.cc new file mode 100644 index 000000000..fd904f465 --- /dev/null +++ b/src/nix/verify.cc @@ -0,0 +1,168 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include <atomic> + +using namespace nix; + +struct CmdVerify : StorePathsCommand +{ + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded; + + CmdVerify() + { + mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag('s', "substituter", {"store-uri"}, "use signatures from specified store", 1, + [&](Strings ss) { substituterUris.push_back(ss.front()); }); + mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + } + + std::string name() override + { + return "verify"; + } + + std::string description() override + { + return "verify the integrity of store paths"; + } + + Examples examples() override + { + return { + Example{ + "To verify the entire Nix store:", + "nix verify --all" + }, + Example{ + "To check whether each path in the closure of Firefox has at least 2 signatures:", + "nix verify -r -n2 --no-contents $(type -p firefox)" + }, + }; + } + + void run(ref<Store> store, Paths storePaths) override + { + std::vector<ref<Store>> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStoreAt(s)); + + auto publicKeys = getDefaultPublicKeys(); + + std::atomic<size_t> done{0}; + std::atomic<size_t> untrusted{0}; + std::atomic<size_t> corrupted{0}; + std::atomic<size_t> failed{0}; + + std::string doneLabel("paths checked"); + std::string untrustedLabel("untrusted"); + std::string corruptedLabel("corrupted"); + std::string failedLabel("failed"); + logger->setExpected(doneLabel, storePaths.size()); + + ThreadPool pool; + + auto doPath = [&](const Path & storePath) { + try { + checkInterrupt(); + + Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + + HashSink sink(info->narHash.type); + store->narFromPath(info->path, sink); + + auto hash = sink.finish(); + + if (hash.first != info->narHash) { + logger->incProgress(corruptedLabel); + corrupted = 1; + printMsg(lvlError, + format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’") + % info->path % printHash(info->narHash) % printHash(hash.first)); + } + + } + + if (!noTrust) { + + bool good = false; + + if (info->ultimate && !sigsNeeded) + good = true; + + else { + + StringSet sigsSeen; + size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t validSigs = 0; + + auto doSigs = [&](StringSet sigs) { + for (auto sig : sigs) { + if (sigsSeen.count(sig)) continue; + sigsSeen.insert(sig); + if (info->checkSignature(publicKeys, sig)) + validSigs++; + } + }; + + doSigs(info->sigs); + + for (auto & store2 : substituters) { + if (validSigs >= actualSigsNeeded) break; + try { + doSigs(store2->queryPathInfo(info->path)->sigs); + } catch (InvalidPath &) { + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + } + } + + if (validSigs >= actualSigsNeeded) + good = true; + } + + if (!good) { + logger->incProgress(untrustedLabel); + untrusted++; + printMsg(lvlError, format("path ‘%s’ is untrusted") % info->path); + } + + } + + logger->incProgress(doneLabel); + done++; + + } catch (Error & e) { + printMsg(lvlError, format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + logger->incProgress(failedLabel); + failed++; + } + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + printMsg(lvlInfo, format("%d paths checked, %d untrusted, %d corrupted, %d failed") + % done % untrusted % corrupted % failed); + + throw Exit( + (corrupted ? 1 : 0) | + (untrusted ? 2 : 0) | + (failed ? 4 : 0)); + } +}; + +static RegisterCommand r1(make_ref<CmdVerify>()); diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index c72d2defa..5f88c595f 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,7 +1,6 @@ source common.sh clearStore -clearManifests clearCache # Create the binary cache. diff --git a/tests/binary-patching.nix b/tests/binary-patching.nix deleted file mode 100644 index 8ed474d1f..000000000 --- a/tests/binary-patching.nix +++ /dev/null @@ -1,18 +0,0 @@ -{ version }: - -with import ./config.nix; - -mkDerivation { - name = "foo-${toString version}"; - builder = builtins.toFile "builder.sh" - '' - mkdir $out - (for ((n = 1; n < 100000; n++)); do echo $n; done) > $out/foo - ${if version != 1 then '' - (for ((n = 100000; n < 110000; n++)); do echo $n; done) >> $out/foo - '' else ""} - ${if version == 3 then '' - echo foobar >> $out/foo - '' else ""} - ''; -} diff --git a/tests/binary-patching.sh b/tests/binary-patching.sh deleted file mode 100644 index 188be109a..000000000 --- a/tests/binary-patching.sh +++ /dev/null @@ -1,61 +0,0 @@ -source common.sh - -clearManifests - -mkdir -p $TEST_ROOT/cache2 $TEST_ROOT/patches - -RESULT=$TEST_ROOT/result - -# Build version 1 and 2 of the "foo" package. -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 \ - $(nix-build -o $RESULT binary-patching.nix --arg version 1) -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest1 - -out2=$(nix-build -o $RESULT binary-patching.nix --arg version 2) -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out2 -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest2 - -out3=$(nix-build -o $RESULT binary-patching.nix --arg version 3) -nix-push --dest $TEST_ROOT/cache2 --manifest --bzip2 $out3 -mv $TEST_ROOT/cache2/MANIFEST $TEST_ROOT/manifest3 - -rm $RESULT - -# Generate binary patches. -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest2 - -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest2 $TEST_ROOT/manifest3 - -grep -q "patch {" $TEST_ROOT/manifest3 - -# Get rid of versions 2 and 3. -nix-store --delete $out2 $out3 - -# Pull the manifest containing the patches. -clearManifests -nix-pull file://$TEST_ROOT/manifest3 - -# Make sure that the download size prediction uses the patches rather -# than the full download. -nix-build -o $RESULT binary-patching.nix --arg version 3 --dry-run 2>&1 | grep -q "0.01 MiB" - -# Now rebuild it. This should use the two patches generated above. -rm -f $TEST_ROOT/var/log/nix/downloads -nix-build -o $RESULT binary-patching.nix --arg version 3 -rm $RESULT -[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 2 ] - -# Add a patch from version 1 directly to version 3. -nix-generate-patches $TEST_ROOT/cache2 $TEST_ROOT/patches \ - file://$TEST_ROOT/patches $TEST_ROOT/manifest1 $TEST_ROOT/manifest3 - -# Rebuild version 3. This should use the direct patch rather than the -# sequence of two patches. -nix-store --delete $out2 $out3 -clearManifests -rm $TEST_ROOT/var/log/nix/downloads -nix-pull file://$TEST_ROOT/manifest3 -nix-build -o $RESULT binary-patching.nix --arg version 3 -[ "$(grep ' patch ' $TEST_ROOT/var/log/nix/downloads | wc -l)" -eq 1 ] diff --git a/tests/check-reqs.sh b/tests/check-reqs.sh index 8b2454915..77689215d 100644 --- a/tests/check-reqs.sh +++ b/tests/check-reqs.sh @@ -6,7 +6,8 @@ nix-build -o $RESULT check-reqs.nix -A test1 (! nix-build -o $RESULT check-reqs.nix -A test2) (! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep1' +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep2' (! nix-build -o $RESULT check-reqs.nix -A test5) (! nix-build -o $RESULT check-reqs.nix -A test6) diff --git a/tests/common.sh.in b/tests/common.sh.in index eb9798a27..9e8962f1a 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -2,7 +2,7 @@ set -e datadir="@datadir@" -export TEST_ROOT=$(pwd)/test-tmp +export TEST_ROOT=${TMPDIR:-/tmp}/nix-test export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. @@ -54,10 +54,6 @@ clearStore() { rm -f "$NIX_STATE_DIR"/gcroots/ref } -clearManifests() { - rm -f $NIX_STATE_DIR/manifests/* -} - clearCache() { rm -rf "$cacheDir" } diff --git a/tests/fallback.sh b/tests/fallback.sh deleted file mode 100644 index f3a6b5051..000000000 --- a/tests/fallback.sh +++ /dev/null @@ -1,20 +0,0 @@ -source common.sh - -clearStore - -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -outPath=$(nix-store -q --fallback "$drvPath") -echo "output path is $outPath" - -# Build with a substitute that fails. This should fail. -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh -if nix-store -r "$drvPath"; then echo unexpected fallback; exit 1; fi - -# Build with a substitute that fails. This should fall back to a source build. -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh -nix-store -r --fallback "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hello World!"; then exit 1; fi diff --git a/tests/filter-source.nix b/tests/filter-source.nix index a620f0fda..907163639 100644 --- a/tests/filter-source.nix +++ b/tests/filter-source.nix @@ -8,5 +8,5 @@ mkDerivation { type != "symlink" && baseNameOf path != "foo" && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); - in builtins.filterSource filter ./test-tmp/filterin; + in builtins.filterSource filter ((builtins.getEnv "TEST_ROOT") + "/filterin"); } diff --git a/tests/install-package.sh b/tests/install-package.sh index 653dfee4c..1916f7271 100644 --- a/tests/install-package.sh +++ b/tests/install-package.sh @@ -1,15 +1,14 @@ source common.sh -# Note: this test expects to be run *after* nix-push.sh. - drvPath=$(nix-instantiate ./dependencies.nix) -outPath=$(nix-store -q $drvPath) +outPath=$(nix-store -r $drvPath) +nix-push --dest $cacheDir $outPath clearStore clearProfiles cat > $TEST_ROOT/foo.nixpkg <<EOF -NIXPKG1 file://$TEST_ROOT/cache/MANIFEST simple $system $drvPath $outPath +NIXPKG1 - simple $system $drvPath $outPath file://$cacheDir EOF nix-install-package --non-interactive -p $profiles/test $TEST_ROOT/foo.nixpkg diff --git a/tests/local.mk b/tests/local.mk index 03f53b44c..7c5a553d3 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,15 +3,14 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - build-hook.sh substitutes.sh substitutes2.sh \ - fallback.sh nix-push.sh gc.sh gc-concurrent.sh nix-pull.sh \ + build-hook.sh nix-push.sh gc.sh gc-concurrent.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ - remote-store.sh export.sh export-graph.sh negative-caching.sh \ - binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ + remote-store.sh export.sh export-graph.sh \ + timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh case-hack.sh \ - check-reqs.sh pass-as-file.sh tarball.sh + check-reqs.sh pass-as-file.sh tarball.sh restricted.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/logging.sh b/tests/logging.sh index 77b2337a9..86f32bade 100644 --- a/tests/logging.sh +++ b/tests/logging.sh @@ -2,16 +2,7 @@ source common.sh clearStore -# Produce an escaped log file. -path=$(nix-build --log-type escapes -vv dependencies.nix --no-out-link 2> $TEST_ROOT/log.esc) - -# Convert it to an XML representation. -nix-log2xml < $TEST_ROOT/log.esc > $TEST_ROOT/log.xml - -# Is this well-formed XML? -if test "$xmllint" != "false"; then - $xmllint --noout $TEST_ROOT/log.xml || fail "malformed XML" -fi +path=$(nix-build dependencies.nix --no-out-link) # Test nix-store -l. [ "$(nix-store -l $path)" = FOO ] diff --git a/tests/negative-caching.nix b/tests/negative-caching.nix deleted file mode 100644 index 10df67a74..000000000 --- a/tests/negative-caching.nix +++ /dev/null @@ -1,21 +0,0 @@ -with import ./config.nix; - -rec { - - fail = mkDerivation { - name = "fail"; - builder = builtins.toFile "builder.sh" "echo FAIL; exit 1"; - }; - - succeed = mkDerivation { - name = "succeed"; - builder = builtins.toFile "builder.sh" "echo SUCCEED; mkdir $out"; - }; - - depOnFail = mkDerivation { - name = "dep-on-fail"; - builder = builtins.toFile "builder.sh" "echo URGH; mkdir $out"; - inputs = [fail succeed]; - }; - -} diff --git a/tests/negative-caching.sh b/tests/negative-caching.sh deleted file mode 100644 index 4217bc38e..000000000 --- a/tests/negative-caching.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore - -set +e - -opts="--option build-cache-failure true --print-build-trace" - -# This build should fail, and the failure should be cached. -log=$(nix-build $opts negative-caching.nix -A fail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "@ build-failed" || fail "no build-failed trace" - -# Do it again. The build shouldn't be tried again. -log=$(nix-build $opts negative-caching.nix -A fail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "FAIL" && fail "failed build not cached" -echo "$log" | grep -q "@ build-failed .* cached" || fail "trace doesn't say cached" - -# Check that --keep-going works properly with cached failures. -log=$(nix-build $opts --keep-going negative-caching.nix -A depOnFail --no-out-link 2>&1) && fail "should fail" -echo "$log" | grep -q "FAIL" && fail "failed build not cached (2)" -echo "$log" | grep -q "@ build-failed .* cached" || fail "trace doesn't say cached (2)" -echo "$log" | grep -q "@ build-succeeded .*-succeed" || fail "didn't keep going" diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index b3442f6a8..c538afd60 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -1,7 +1,6 @@ source common.sh clearProfiles -clearManifests rm -f $TEST_ROOT/.nix-channels @@ -45,7 +44,6 @@ nix-env -i dependencies clearProfiles -clearManifests rm -f $TEST_ROOT/.nix-channels # Test updating from a tarball diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index 1418c6589..0e42cc0a3 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -4,7 +4,7 @@ with import <nixpkgs/nixos/lib/testing.nix> { inherit system; }; -makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; pkgC = pkgs.hello; in { +makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { nodes = { client = diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 3586a7efc..db1edd73e 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -1,10 +1,11 @@ source common.sh home=$TEST_ROOT/home +user=$(whoami) rm -rf $home mkdir -p $home -HOME=$home $SHELL -e -c ". ../scripts/nix-profile.sh" -HOME=$home $SHELL -e -c ". ../scripts/nix-profile.sh" # test idempotency +HOME=$home USER=$user $SHELL -e -c ". ../scripts/nix-profile.sh" +HOME=$home USER=$user $SHELL -e -c ". ../scripts/nix-profile.sh" # test idempotency [ -L $home/.nix-profile ] [ -e $home/.nix-channels ] diff --git a/tests/nix-pull.sh b/tests/nix-pull.sh deleted file mode 100644 index 87239948c..000000000 --- a/tests/nix-pull.sh +++ /dev/null @@ -1,33 +0,0 @@ -source common.sh - -pullCache () { - echo "pulling cache..." - nix-pull file://$TEST_ROOT/cache/MANIFEST -} - -clearStore -clearManifests -pullCache - -drvPath=$(nix-instantiate dependencies.nix) -outPath=$(nix-store -q $drvPath) - -echo "building $outPath using substitutes..." -nix-store -r $outPath - -cat $outPath/input-2/bar - -clearStore -clearManifests -pullCache - -echo "building $drvPath using substitutes..." -nix-store -r $drvPath - -cat $outPath/input-2/bar - -# Check that the derivers are set properly. -test $(nix-store -q --deriver "$outPath") = "$drvPath" -nix-store -q --deriver $(readLink $outPath/input-2) | grep -q -- "-input-2.drv" - -clearManifests diff --git a/tests/referrers.sh b/tests/referrers.sh index 0a1c86e0a..5c1ef20cf 100644 --- a/tests/referrers.sh +++ b/tests/referrers.sh @@ -4,7 +4,7 @@ clearStore max=500 -reference=$NIX_STORE_DIR/abcdef +reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa touch $reference (echo $reference && echo && echo 0) | nix-store --register-validity @@ -12,9 +12,9 @@ echo "making registration..." set +x for ((n = 0; n < $max; n++)); do - storePath=$NIX_STORE_DIR/$n + storePath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$n echo -n > $storePath - ref2=$NIX_STORE_DIR/$((n+1)) + ref2=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$((n+1)) if test $((n+1)) = $max; then ref2=$reference fi @@ -30,7 +30,7 @@ echo "collecting garbage..." ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref nix-store --gc -if [ -n "$(type -p sqlite3)" -a "$(sqlite3 ./test-tmp/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then +if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_DB_DIR/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then echo "referrers not cleaned up" exit 1 fi diff --git a/tests/remote-store.sh b/tests/remote-store.sh index 8312424f0..b3908717a 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -1,7 +1,6 @@ source common.sh clearStore -clearManifests startDaemon diff --git a/tests/restricted.sh b/tests/restricted.sh new file mode 100644 index 000000000..19096a9f8 --- /dev/null +++ b/tests/restricted.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore + +nix-instantiate --option restrict-eval true --eval -E '1 + 2' +(! nix-instantiate --option restrict-eval true ./simple.nix) +nix-instantiate --option restrict-eval true ./simple.nix -I src=. +nix-instantiate --option restrict-eval true ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh + +(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix') +nix-instantiate --option restrict-eval true --eval -E 'builtins.readFile ./simple.nix' -I src=.. + +(! nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost') +nix-instantiate --option restrict-eval true --eval -E 'builtins.readDir ../src/boost' -I src=../src + +(! nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') +nix-instantiate --option restrict-eval true --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. + diff --git a/tests/secure-drv-outputs.sh b/tests/secure-drv-outputs.sh index 4888123da..50a9c4428 100644 --- a/tests/secure-drv-outputs.sh +++ b/tests/secure-drv-outputs.sh @@ -5,7 +5,6 @@ source common.sh clearStore -clearManifests startDaemon diff --git a/tests/substituter.sh b/tests/substituter.sh deleted file mode 100755 index 9aab295de..000000000 --- a/tests/substituter.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/sh -e -echo -echo substituter args: $* >&2 - -if test $1 = "--query"; then - while read cmd args; do - echo "CMD = $cmd, ARGS = $args" >&2 - if test "$cmd" = "have"; then - for path in $args; do - read path - if grep -q "$path" $TEST_ROOT/sub-paths; then - echo $path - fi - done - echo - elif test "$cmd" = "info"; then - for path in $args; do - echo $path - echo "" # deriver - echo 0 # nr of refs - echo $((1 * 1024 * 1024)) # download size - echo $((2 * 1024 * 1024)) # nar size - done - echo - else - echo "bad command $cmd" - exit 1 - fi - done -elif test $1 = "--substitute"; then - mkdir $2 - echo "Hallo Wereld" > $2/hello - echo # no expected hash -else - echo "unknown substituter operation" - exit 1 -fi diff --git a/tests/substituter2.sh b/tests/substituter2.sh deleted file mode 100755 index 5d1763599..000000000 --- a/tests/substituter2.sh +++ /dev/null @@ -1,33 +0,0 @@ -#! /bin/sh -e -echo -echo substituter2 args: $* >&2 - -if test $1 = "--query"; then - while read cmd args; do - if test "$cmd" = have; then - for path in $args; do - if grep -q "$path" $TEST_ROOT/sub-paths; then - echo $path - fi - done - echo - elif test "$cmd" = info; then - for path in $args; do - echo $path - echo "" # deriver - echo 0 # nr of refs - echo 0 # download size - echo 0 # nar size - done - echo - else - echo "bad command $cmd" - exit 1 - fi - done -elif test $1 = "--substitute"; then - exit 1 -else - echo "unknown substituter operation" - exit 1 -fi diff --git a/tests/substitutes.sh b/tests/substitutes.sh deleted file mode 100644 index 0c6adf260..000000000 --- a/tests/substitutes.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -clearStore - -# Instantiate. -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -# Find the output path. -outPath=$(nix-store -qvv "$drvPath") -echo "output path is $outPath" - -echo $outPath > $TEST_ROOT/sub-paths - -export NIX_SUBSTITUTERS=$(pwd)/substituter.sh - -nix-store -r "$drvPath" --dry-run 2>&1 | grep -q "1.00 MiB.*2.00 MiB" - -nix-store -rvv "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi diff --git a/tests/substitutes2.sh b/tests/substitutes2.sh deleted file mode 100644 index bd914575c..000000000 --- a/tests/substitutes2.sh +++ /dev/null @@ -1,21 +0,0 @@ -source common.sh - -clearStore - -# Instantiate. -drvPath=$(nix-instantiate simple.nix) -echo "derivation is $drvPath" - -# Find the output path. -outPath=$(nix-store -qvvvvv "$drvPath") -echo "output path is $outPath" - -echo $outPath > $TEST_ROOT/sub-paths - -# First try a substituter that fails, then one that succeeds -export NIX_SUBSTITUTERS=$(pwd)/substituter2.sh:$(pwd)/substituter.sh - -nix-store -j0 -rvv "$drvPath" - -text=$(cat "$outPath"/hello) -if test "$text" != "Hallo Wereld"; then echo "wrong substitute output: $text"; exit 1; fi diff --git a/tests/tarball.sh b/tests/tarball.sh index cb5258a9e..254c4b626 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -16,8 +16,14 @@ tarball=$TEST_ROOT/tarball.tar.xz nix-env -f file://$tarball -qa --out-path | grep -q dependencies -nix-build file://$tarball +nix-build -o $TMPDIR/result file://$tarball -nix-build '<foo>' -I foo=file://$tarball +nix-build -o $TMPDIR/result '<foo>' -I foo=file://$tarball -nix-build -E "import (fetchTarball file://$tarball)" +nix-build -o $TMPDIR/result -E "import (fetchTarball file://$tarball)" + +nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz +nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz +(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz) + +nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=. |