diff options
152 files changed, 4100 insertions, 1174 deletions
@@ -1 +1 @@ -2.3
\ No newline at end of file +2.4
\ No newline at end of file @@ -19,11 +19,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include config.h -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/Makefile.config.in b/Makefile.config.in index 59730b646..7e3b35b98 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -5,10 +5,11 @@ CC = @CC@ CFLAGS = @CFLAGS@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ +LDFLAGS = @LDFLAGS@ ENABLE_S3 = @ENABLE_S3@ HAVE_SODIUM = @HAVE_SODIUM@ -HAVE_READLINE = @HAVE_READLINE@ HAVE_SECCOMP = @HAVE_SECCOMP@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ LIBCURL_LIBS = @LIBCURL_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ PACKAGE_NAME = @PACKAGE_NAME@ @@ -1,3 +1,5 @@ +[](https://opencollective.com/nixos) + Nix, the purely functional package manager ------------------------------------------ diff --git a/configure.ac b/configure.ac index f5b1614f1..ebe6d4267 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,5 @@ AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_CONFIG_MACRO_DIRS([m4]) AC_CONFIG_SRCDIR(README.md) AC_CONFIG_AUX_DIR(config) @@ -42,27 +43,21 @@ esac AC_MSG_RESULT($system) AC_SUBST(system) -AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier (`cpu-os')]) +AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var -# Solaris-specific stuff. -AC_STRUCT_DIRENT_D_TYPE -if test "$sys_name" = sunos; then - # Solaris requires -lsocket -lnsl for network functions - LIBS="-lsocket -lnsl $LIBS" -fi - - -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AC_PROG_CPP -AX_CXX_COMPILE_STDCXX_17 +AX_CXX_COMPILE_STDCXX_17([noext], [mandatory]) AC_CHECK_TOOL([AR], [ar]) @@ -70,6 +65,14 @@ AC_CHECK_TOOL([AR], [ar]) AC_SYS_LARGEFILE +# Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE +if test "$sys_name" = sunos; then + # Solaris requires -lsocket -lnsl for network functions + LIBS="-lsocket -lnsl $LIBS" +fi + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -145,6 +148,28 @@ AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], AC_SUBST(storedir) +# Look for boost, a required dependency. +# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, +# and CPPFLAGS are not passed to the C++ compiler automatically. +# Thus we append the returned CPPFLAGS to the CXXFLAGS here. +AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) +# For unknown reasons, setting this directly in the ACTION-IF-FOUND above +# ends up with LDFLAGS being empty, so we set it afterwards. +LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" + +# On some platforms, new-style atomics need a helper library +AC_MSG_CHECKING(whether -latomic is needed) +AC_LINK_IFELSE([AC_LANG_SOURCE([[ +#include <stdint.h> +uint64_t v; +int main() { + return (int)__atomic_load_n(&v, __ATOMIC_ACQUIRE); +}]])], GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=no, GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC=yes) +AC_MSG_RESULT($GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC) +if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then + LIBS="-latomic $LIBS" +fi + # Look for OpenSSL, a required dependency. PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) @@ -164,7 +189,16 @@ PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CX PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) # Look for editline, a required dependency. -PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"]) +# The the libeditline.pc file was added only in libeditline >= 1.15.2, +# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, +# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for +# editline.h when the pkg-config approach fails. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ + AC_CHECK_HEADERS([editline.h], [true], + [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) + AC_SEARCH_LIBS([readline read_history], [editline], [], + [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) +]) # Look for libsodium, an optional dependency. PKG_CHECK_MODULES([SODIUM], [libsodium], diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py new file mode 100755 index 000000000..f5602c95c --- /dev/null +++ b/contrib/stack-collapse.py @@ -0,0 +1,38 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls '<nixpkgs>' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/corepkgs/unpack-channel.nix b/corepkgs/unpack-channel.nix index a654db40e..d39a20637 100644 --- a/corepkgs/unpack-channel.nix +++ b/corepkgs/unpack-channel.nix @@ -18,21 +18,17 @@ let if [ * != $channelName ]; then mv * $out/$channelName fi - if [ -n "$binaryCacheURL" ]; then - mkdir $out/binary-caches - echo -n "$binaryCacheURL" > $out/binary-caches/$channelName - fi ''; in -{ name, channelName, src, binaryCacheURL ? "" }: +{ name, channelName, src }: derivation { system = builtins.currentSystem; builder = shell; args = [ "-e" builder ]; - inherit name channelName src binaryCacheURL; + inherit name channelName src; PATH = "${nixBinDir}:${coreutils}"; diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index c304367aa..871b7eb1d 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -7,6 +7,8 @@ <title>Advanced Topics</title> <xi:include href="distributed-builds.xml" /> +<xi:include href="cores-vs-jobs.xml" /> <xi:include href="diff-hook.xml" /> +<xi:include href="post-build-hook.xml" /> </part> diff --git a/doc/manual/advanced-topics/cores-vs-jobs.xml b/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 000000000..eba645faf --- /dev/null +++ b/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="chap-tuning-cores-and-jobs"> + +<title>Tuning Cores and Jobs</title> + +<para>Nix has two relevant settings with regards to how your CPU cores +will be utilized: <xref linkend="conf-cores" /> and +<xref linkend="conf-max-jobs" />. This chapter will talk about what +they are, how they interact, and their configuration trade-offs.</para> + +<variablelist> + <varlistentry> + <term><xref linkend="conf-max-jobs" /></term> + <listitem><para> + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + </para></listitem> + </varlistentry> + <varlistentry> + <term><xref linkend="conf-cores" /></term> + <listitem><para> + Suggests how many cores each derivation should use. Similar to + <command>make -j</command>. + </para></listitem> + </varlistentry> +</variablelist> + +<para>The <xref linkend="conf-cores" /> setting determines the value of +<envar>NIX_BUILD_CORES</envar>. <envar>NIX_BUILD_CORES</envar> is equal +to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" /> +equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar> +will be the total number of cores in the system.</para> + +<para>The total number of consumed cores is a simple multiplication, +<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para> + +<para>The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores:</para> + +<table> + <caption>Balancing 24 Build Cores</caption> + <thead> + <tr> + <th><xref linkend="conf-max-jobs" /></th> + <th><xref linkend="conf-cores" /></th> + <th><envar>NIX_BUILD_CORES</envar></th> + <th>Maximum Processes</th> + <th>Result</th> + </tr> + </thead> + <tbody> + <tr> + <td>1</td> + <td>24</td> + <td>24</td> + <td>24</td> + <td> + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. + </td> + </tr> + + <tr> + <td>4</td> + <td>6</td> + <td>6</td> + <td>24</td> + <td> + Four derivations will be built at once, each given access to + six cores. + </td> + </tr> + <tr> + <td>12</td> + <td>6</td> + <td>6</td> + <td>72</td> + <td> + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. + </td> + </tr> + <tr> + <td>24</td> + <td>1</td> + <td>1</td> + <td>24</td> + <td> + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. + </td> + </tr> + <tr> + <td>24</td> + <td>0</td> + <td>24</td> + <td>576</td> + <td> + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. + </td> + </tr> + </tbody> +</table> + +<para>It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +<envar>NIX_BUILD_CORES</envar> environment variable.</para> + +</chapter> diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000..3dc43ee79 --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="chap-post-build-hook" + version="5.0" + > + +<title>Using the <xref linkend="conf-post-build-hook" /></title> +<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle> + + +<section xml:id="chap-post-build-hook-caveats"> + <title>Implementation Caveats</title> + <para>Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases.</para> + + <para>The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails.</para> + + <para>Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable.</para> + + <para>A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop.</para> +</section> + +<section> + <title>Prerequisites</title> + + <para> + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + <xref linkend="ssec-s3-substituter-authenticated-writes" />, and + that the <literal>root</literal> user's default AWS profile can + upload to the bucket. + </para> +</section> + +<section> + <title>Set up a Signing Key</title> + <para>Use <command>nix-store --generate-binary-cache-key</command> to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths.</para> + + <screen> +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +</screen> + +<para>Then, add the public key and the cache URL to your +<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" /> +and <xref linkend="conf-substituters" /> like:</para> + +<programlisting> +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +</programlisting> + +<para>we will restart the Nix daemon a later step.</para> +</section> + +<section> + <title>Implementing the build hook</title> + <para>Write the following script to + <filename>/etc/nix/upload-to-cache.sh</filename>: + </para> + + <programlisting> +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS +</programlisting> + + <note> + <title>Should <literal>$OUT_PATHS</literal> be quoted?</title> + <para> + The <literal>$OUT_PATHS</literal> variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to <command>nix sign-paths</command>. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The <command>set -f</command> + disables globbing in the shell. + </para> + </note> + <para> + Then make sure the hook program is executable by the <literal>root</literal> user: + <screen> +# chmod +x /etc/nix/upload-to-cache.sh +</screen></para> +</section> + +<section> + <title>Updating Nix Configuration</title> + + <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook, + by adding the following configuration snippet at the end:</para> + + <programlisting> +post-build-hook = /etc/nix/upload-to-cache.sh +</programlisting> + +<para>Then, restart the <command>nix-daemon</command>.</para> +</section> + +<section> + <title>Testing</title> + + <para>Build any derivation, for example:</para> + + <screen> +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +</screen> + + <para>Then delete the path from the store, and try substituting it from the binary cache:</para> + <screen> +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +</screen> + +<para>Now, copy the path back from the cache:</para> +<screen> +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example +</screen> +</section> +<section> + <title>Conclusion</title> + <para> + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + </para> + + <para> + Before deploying this to production, be sure to consider the + implementation caveats in <xref linkend="chap-post-build-hook-caveats" />. + </para> +</section> +</chapter> diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 24fbf28cf..48dce7c95 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -238,8 +238,9 @@ false</literal>.</para> linkend='opt-cores'>--cores</option> command line switch and defaults to <literal>1</literal>. The value <literal>0</literal> means that the builder should use all available CPU cores in the - system.</para></listitem> + system.</para> + <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para></listitem> </varlistentry> <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term> @@ -432,7 +433,7 @@ builtins.fetchurl { <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term> <listitem><para>If <literal>false</literal> (default), derivations - are not stored in Nix user environments. That is, the derivation + are not stored in Nix user environments. That is, the derivations of any build-time-only dependencies may be garbage-collected.</para> <para>If <literal>true</literal>, when you add a Nix derivation to @@ -482,8 +483,10 @@ builtins.fetchurl { <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term> - <listitem><para>This option defines after how many free bytes to stop collecting - garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem> + <listitem><para>When a garbage collection is triggered by the + <literal>min-free</literal> option, it stops as soon as + <literal>max-free</literal> bytes are available. The default is + infinity (i.e. delete all garbage).</para></listitem> </varlistentry> @@ -498,7 +501,10 @@ builtins.fetchurl { regardless). It can be overridden using the <option linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>) - command line switch.</para></listitem> + command line switch.</para> + + <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para> + </listitem> </varlistentry> <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term> @@ -524,9 +530,11 @@ builtins.fetchurl { <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term> <listitem> - <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix - will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk. - A value of <literal>0</literal> (the default) means that this feature is disabled.</para> + <para>When free disk space in <filename>/nix/store</filename> + drops below <literal>min-free</literal> during a build, Nix + performs a garbage-collection until <literal>max-free</literal> + bytes are available or there is no more garbage. A value of + <literal>0</literal> (the default) disables this feature.</para> </listitem> </varlistentry> @@ -656,6 +664,62 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-post-build-hook"> + <term><literal>post-build-hook</literal></term> + <listitem> + <para>Optional. The path to a program to execute after each build.</para> + + <para>This option is only settable in the global + <filename>nix.conf</filename>, or on the command line by trusted + users.</para> + + <para>When using the nix-daemon, the daemon executes the hook as + <literal>root</literal>. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build.</para> + + <itemizedlist> + <listitem><para>The hook executes after an evaluation-time build.</para></listitem> + <listitem><para>The hook does not execute on substituted paths.</para></listitem> + <listitem><para>The hook's output always goes to the user's terminal.</para></listitem> + <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem> + <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem> + </itemizedlist> + + <para>The program executes with no arguments. The program's environment + contains the following environment variables:</para> + + <variablelist> + <varlistentry> + <term><envar>DRV_PATH</envar></term> + <listitem> + <para>The derivation for the built paths.</para> + <para>Example: + <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><envar>OUT_PATHS</envar></term> + <listitem> + <para>Output paths of the built derivation, separated by a space character.</para> + <para>Example: + <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>See <xref linkend="chap-post-build-hook" /> for an example + implementation.</para> + + </listitem> + </varlistentry> + <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term> <listitem><para>How many times to repeat builds to check whether @@ -809,6 +873,14 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term> + <listitem> + <para>The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration.</para> + </listitem> + </varlistentry> + <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by @@ -864,6 +936,31 @@ requiredSystemFeatures = [ "kvm" ]; </varlistentry> + <varlistentry xml:id="conf-tarball-ttl"><term><literal>tarball-ttl</literal></term> + + <listitem> + <para>Default: <literal>3600</literal> seconds.</para> + + <para>The number of seconds a downloaded tarball is considered + fresh. If the cached tarball is stale, Nix will check whether + it is still up to date using the ETag header. Nix will download + a new version if the ETag header is unsupported, or the + cached ETag doesn't match. + </para> + + <para>Setting the TTL to <literal>0</literal> forces Nix to always + check if the tarball is up to date.</para> + + <para>Nix caches tarballs in + <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para> + + <para>Files fetched via <envar>NIX_PATH</envar>, + <function>fetchGit</function>, <function>fetchMercurial</function>, + <function>fetchTarball</function>, and <function>fetchurl</function> + respect this TTL. + </para> + </listitem> + </varlistentry> <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term> @@ -884,6 +981,34 @@ requiredSystemFeatures = [ "kvm" ]; </varlistentry> + <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term> + + <listitem> + + <para>Default: <literal>false</literal>.</para> + + <para>If set to <literal>true</literal>, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit.</para> + + <informalexample><screen> +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 +</screen></informalexample> + + <para>The <literal>undefined position</literal> means the function + call is a builtin.</para> + + <para>Use the <literal>contrib/stack-collapse.py</literal> script + distributed with the Nix source code to convert the trace logs + in to a format suitable for <command>flamegraph.pl</command>.</para> + + </listitem> + + </varlistentry> + <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term> <listitem><para>A whitespace-separated list of public keys. When diff --git a/doc/manual/command-ref/nix-channel.xml b/doc/manual/command-ref/nix-channel.xml index ff4021a76..9fea59167 100644 --- a/doc/manual/command-ref/nix-channel.xml +++ b/doc/manual/command-ref/nix-channel.xml @@ -31,12 +31,11 @@ <refsection><title>Description</title> -<para>A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing both a set of Nix -expressions and a pointer to a binary cache. <phrase -condition="manual">See also <xref linkend="sec-channels" -/>.</phrase></para> +<para>A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. <phrase condition="manual">See also <xref +linkend="sec-channels" />.</phrase></para> <para>This command has the following operations: @@ -112,13 +111,13 @@ $ nix-env -iA nixpkgs.hello</screen> <para>You can revert channel updates using <option>--rollback</option>:</para> <screen> -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.527.0e935f1" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' "14.04.526.dbadfad" </screen> @@ -172,18 +171,6 @@ following files:</para> </varlistentry> - <varlistentry><term><filename>binary-cache-url</filename></term> - - <listitem><para>A file containing the URL to a binary cache (such - as <uri>https://cache.nixos.org</uri>). Nix will automatically - check this cache for pre-built binaries, if the user has - sufficient rights to add binary caches. For instance, in a - multi-user Nix setup, the binary caches provided by the channels - of the root user are used automatically, but caches corresponding - to the channels of non-root users are ignored.</para></listitem> - - </varlistentry> - </variablelist> </refsection> diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 56c466268..d257a5e49 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -221,31 +221,53 @@ also <xref linkend="sec-common-options" />.</phrase></para> <varlistentry><term><filename>~/.nix-defexpr</filename></term> - <listitem><para>A directory that contains the default Nix + <listitem><para>The source for the default Nix expressions used by the <option>--install</option>, <option>--upgrade</option>, and <option>--query - --available</option> operations to obtain derivations. The + --available</option> operations to obtain derivations. The <option>--file</option> option may be used to override this default.</para> - <para>The Nix expressions in this directory are combined into a - single set, with each file as an attribute that has the name of - the file. Thus, if <filename>~/.nix-defexpr</filename> contains - two files, <filename>foo</filename> and <filename>bar</filename>, + <para>If <filename>~/.nix-defexpr</filename> is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory + containing a <filename>default.nix</filename> file, that file + is loaded as in the above paragraph.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory without + a <filename>default.nix</filename> file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + </para> + + <para>For example, if <filename>~/.nix-defexpr</filename> contains + two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>, then the default Nix expression will essentially be <programlisting> { - foo = import ~/.nix-defexpr/foo; - bar = import ~/.nix-defexpr/bar; + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; }</programlisting> </para> + <para>The file <filename>manifest.nix</filename> is always ignored. + Subdirectories without a <filename>default.nix</filename> file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression.</para> + <para>The command <command>nix-channel</command> places symlinks to the downloaded Nix expressions from each subscribed channel in this directory.</para> - </listitem> </varlistentry> @@ -637,7 +659,7 @@ upgrading `mozilla-1.2' to `mozilla-1.4'</screen> <literal>gcc-3.3.1</literal> are split into two parts: the package name (<literal>gcc</literal>), and the version (<literal>3.3.1</literal>). The version part starts after the first -dash not following by a letter. <varname>x</varname> is considered an +dash not followed by a letter. <varname>x</varname> is considered an upgrade of <varname>y</varname> if their package names match, and the version of <varname>y</varname> is higher that that of <varname>x</varname>.</para> @@ -1348,10 +1370,13 @@ profile. The generations can be a list of generation numbers, the special value <literal>old</literal> to delete all non-current generations, a value such as <literal>30d</literal> to delete all generations older than the specified number of days (except for the -generation that was active at that point in time), or a value such as. -<literal>+5</literal> to only keep the specified items older than the -current generation. Periodically deleting old generations is important -to make garbage collection effective.</para> +generation that was active at that point in time), or a value such as +<literal>+5</literal> to keep the last <literal>5</literal> generations +ignoring any newer than current, e.g., if <literal>30</literal> is the current +generation <literal>+5</literal> will delete generation <literal>25</literal> +and all older generations. +Periodically deleting old generations is important to make garbage collection +effective.</para> </refsection> diff --git a/doc/manual/command-ref/nix-prefetch-url.xml b/doc/manual/command-ref/nix-prefetch-url.xml index 8ef748c74..621ded72e 100644 --- a/doc/manual/command-ref/nix-prefetch-url.xml +++ b/doc/manual/command-ref/nix-prefetch-url.xml @@ -53,7 +53,7 @@ avoided.</para> <para>If <replaceable>hash</replaceable> is specified, then a download is not performed if the Nix store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an -error if signaled if the actual hash of the file does not match the +error is signaled if the actual hash of the file does not match the specified hash.</para> <para>This command prints the hash on standard output. Additionally, diff --git a/doc/manual/command-ref/nix-store.xml b/doc/manual/command-ref/nix-store.xml index d73cb92ee..113a3c2e4 100644 --- a/doc/manual/command-ref/nix-store.xml +++ b/doc/manual/command-ref/nix-store.xml @@ -215,6 +215,48 @@ printed.)</para> </variablelist> +<para>Special exit codes:</para> + +<variablelist> + + <varlistentry><term><literal>100</literal></term> + <listitem><para>Generic build failure, the builder process + returned with a non-zero exit code.</para></listitem> + </varlistentry> + + <varlistentry><term><literal>101</literal></term> + <listitem><para>Build timeout, the build was aborted because it + did not complete within the specified <link + linkend='conf-timeout'><literal>timeout</literal></link>. + </para></listitem> + </varlistentry> + + <varlistentry><term><literal>102</literal></term> + <listitem><para>Hash mismatch, the build output was rejected + because it does not match the specified <link + linkend="fixed-output-drvs"><varname>outputHash</varname></link>. + </para></listitem> + </varlistentry> + + <varlistentry><term><literal>104</literal></term> + <listitem><para>Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable.</para> + </listitem> + </varlistentry> + +</variablelist> + +<para>With the <option>--keep-going</option> flag it's possible for +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. <screen> +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +</screen></para> + </refsection> @@ -883,6 +925,60 @@ $ nix-store --add ./foo.c </refsection> +<!--######################################################################--> + +<refsection><title>Operation <option>--add-fixed</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg><option>--recursive</option></arg> + <arg choice='plain'><option>--add-fixed</option></arg> + <arg choice='plain'><replaceable>algorithm</replaceable></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--add-fixed</option> adds the specified paths to +the Nix store. Unlike <option>--add</option> paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. +</para> + +<para>This operation has the following options: + +<variablelist> + + <varlistentry><term><option>--recursive</option></term> + + <listitem><para> + Use recursive instead of flat hashing mode, used when adding directories + to the store. + </para></listitem> + + </varlistentry> + +</variablelist> + +</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz</screen> + +</refsection> + +</refsection> + <!--######################################################################--> diff --git a/doc/manual/expressions/builtins.xml b/doc/manual/expressions/builtins.xml index a87639a07..f39b393bf 100644 --- a/doc/manual/expressions/builtins.xml +++ b/doc/manual/expressions/builtins.xml @@ -170,18 +170,6 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> </varlistentry> - <varlistentry xml:id='builtin-splitVersion'> - <term><function>builtins.splitVersion</function> - <replaceable>s</replaceable></term> - - <listitem><para>Split a string representing a version into its - components, by the same version splitting logic underlying the - version comparison in <link linkend="ssec-version-comparisons"> - <command>nix-env -u</command></link>.</para></listitem> - - </varlistentry> - - <varlistentry xml:id='builtin-concatLists'> <term><function>builtins.concatLists</function> <replaceable>lists</replaceable></term> @@ -301,7 +289,7 @@ if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> <listitem><para>Return element <replaceable>n</replaceable> from the list <replaceable>xs</replaceable>. Elements are counted - starting from 0. A fatal error occurs in the index is out of + starting from 0. A fatal error occurs if the index is out of bounds.</para></listitem> </varlistentry> @@ -347,7 +335,7 @@ stdenv.mkDerivation { … } You can change the cache timeout either on the command line with <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or in the Nix configuration file with this option: - <literal>tarball-ttl <replaceable>number of seconds to cache</replaceable></literal>. + <literal><xref linkend="conf-tarball-ttl" /> <replaceable>number of seconds to cache</replaceable></literal>. </para> <para>Note that when obtaining the hash with <varname>nix-prefetch-url @@ -425,6 +413,13 @@ stdenv.mkDerivation { … } This is often a branch or tag name. Defaults to <literal>HEAD</literal>. </para> + + <para> + By default, the <varname>ref</varname> value is prefixed + with <literal>refs/heads/</literal>. As of Nix 2.3.0 + Nix will not prefix <literal>refs/heads/</literal> if + <varname>ref</varname> starts with <literal>refs/</literal>. + </para> </listitem> </varlistentry> </variablelist> @@ -439,6 +434,14 @@ stdenv.mkDerivation { … } </example> <example> + <title>Fetching an arbitrary ref</title> + <programlisting>builtins.fetchGit { + url = "https://github.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +}</programlisting> + </example> + + <example> <title>Fetching a repository's specific commit on an arbitrary branch</title> <para> If the revision you're looking for is in the default branch @@ -484,11 +487,8 @@ stdenv.mkDerivation { … } <title>Fetching a tag</title> <programlisting>builtins.fetchGit { url = "https://github.com/nixos/nix.git"; - ref = "tags/1.9"; + ref = "refs/tags/1.9"; }</programlisting> - <note><para>Due to a bug (<link - xlink:href="https://github.com/NixOS/nix/issues/2385">#2385</link>), - only non-annotated tags can be fetched.</para></note> </example> <example> @@ -498,7 +498,7 @@ stdenv.mkDerivation { … } fetch the latest version of a remote branch. </para> <note><para>Nix will refetch the branch in accordance to - <option>tarball-ttl</option>.</para></note> + <xref linkend="conf-tarball-ttl" />.</para></note> <note><para>This behavior is disabled in <emphasis>Pure evaluation mode</emphasis>.</para></note> <programlisting>builtins.fetchGit { @@ -746,6 +746,11 @@ builtins.genList (x: x * x) 5 separate file, and use it from Nix expressions in other files.</para> + <note><para>Unlike some languages, <function>import</function> is a regular + function in Nix. Paths using the angle bracket syntax (e.g., <function> + import</function> <replaceable><foo></replaceable>) are normal path + values (see <xref linkend='ssec-values' />).</para></note> + <para>A Nix expression loaded by <function>import</function> must not contain any <emphasis>free variables</emphasis> (identifiers that are not defined in the Nix expression itself and are not @@ -1260,6 +1265,19 @@ Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. </para></listitem> </varlistentry> + + <varlistentry xml:id='builtin-splitVersion'> + <term><function>builtins.splitVersion</function> + <replaceable>s</replaceable></term> + + <listitem><para>Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in <link linkend="ssec-version-comparisons"> + <command>nix-env -u</command></link>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-stringLength'> <term><function>builtins.stringLength</function> <replaceable>e</replaceable></term> @@ -1453,7 +1471,7 @@ in foo</programlisting> <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem> <listitem><para>An integer.</para></listitem> <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem> - <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>.</para></listitem> + <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>).</para></listitem> <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem> </itemizedlist> </listitem> @@ -1592,12 +1610,18 @@ stdenv.mkDerivation (rec { <term><function>builtins.tryEval</function> <replaceable>e</replaceable></term> - <listitem><para>Try to evaluate <replaceable>e</replaceable>. + <listitem><para>Try to shallowly evaluate <replaceable>e</replaceable>. Return a set containing the attributes <literal>success</literal> (<literal>true</literal> if <replaceable>e</replaceable> evaluated successfully, <literal>false</literal> if an error was thrown) and <literal>value</literal>, equalling <replaceable>e</replaceable> - if successful and <literal>false</literal> otherwise. + if successful and <literal>false</literal> otherwise. Note that this + doesn't evaluate <replaceable>e</replaceable> deeply, so + <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success + </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq + </literal> one can get the expected result: <literal>let e = { x = throw ""; + }; in (builtins.tryEval (builtins.deepSeq e e)).success</literal> will be + <literal>false</literal>. </para></listitem> </varlistentry> diff --git a/doc/manual/expressions/language-constructs.xml b/doc/manual/expressions/language-constructs.xml index f961ed921..0d0cbbe15 100644 --- a/doc/manual/expressions/language-constructs.xml +++ b/doc/manual/expressions/language-constructs.xml @@ -43,7 +43,7 @@ encountered</quote>).</para></footnote>.</para> <simplesect xml:id="sect-let-expressions"><title>Let-expressions</title> -<para>A let-expression allows you define local variables for an +<para>A let-expression allows you to define local variables for an expression. For instance, <programlisting> @@ -217,7 +217,25 @@ but can also be written as: ellipsis(<literal>...</literal>) as you can access attribute names as <literal>a</literal>, using <literal>args.a</literal>, which was given as an additional attribute to the function. - </para></listitem> + </para> + + <warning> + <para> + The <literal>args@</literal> expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in <literal>args</literal>. + </para> + <para> + For instance +<programlisting> +let + function = args@{ a ? 23, ... }: args; +in + function {} +</programlisting> + will evaluate to an empty attribute set. + </para> + </warning></listitem> </itemizedlist> diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index f1f750934..4f11bf529 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -15,13 +15,16 @@ weakest binding).</para> <tgroup cols='3'> <thead> <row> + <entry>Name</entry> <entry>Syntax</entry> <entry>Associativity</entry> <entry>Description</entry> + <entry>Precedence</entry> </row> </thead> <tbody> <row> + <entry>Select</entry> <entry><replaceable>e</replaceable> <literal>.</literal> <replaceable>attrpath</replaceable> [ <literal>or</literal> <replaceable>def</replaceable> ] @@ -33,19 +36,25 @@ weakest binding).</para> dot-separated list of attribute names.) If the attribute doesn’t exist, return <replaceable>def</replaceable> if provided, otherwise abort evaluation.</entry> + <entry>1</entry> </row> <row> + <entry>Application</entry> <entry><replaceable>e1</replaceable> <replaceable>e2</replaceable></entry> <entry>left</entry> <entry>Call function <replaceable>e1</replaceable> with argument <replaceable>e2</replaceable>.</entry> + <entry>2</entry> </row> <row> + <entry>Arithmetic Negation</entry> <entry><literal>-</literal> <replaceable>e</replaceable></entry> <entry>none</entry> <entry>Arithmetic negation.</entry> + <entry>3</entry> </row> <row> + <entry>Has Attribute</entry> <entry><replaceable>e</replaceable> <literal>?</literal> <replaceable>attrpath</replaceable></entry> <entry>none</entry> @@ -53,34 +62,69 @@ weakest binding).</para> the attribute denoted by <replaceable>attrpath</replaceable>; return <literal>true</literal> or <literal>false</literal>.</entry> + <entry>4</entry> </row> <row> + <entry>List Concatenation</entry> <entry><replaceable>e1</replaceable> <literal>++</literal> <replaceable>e2</replaceable></entry> <entry>right</entry> <entry>List concatenation.</entry> + <entry>5</entry> </row> <row> + <entry>Multiplication</entry> <entry> <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>, + </entry> + <entry>left</entry> + <entry>Arithmetic multiplication.</entry> + <entry>6</entry> + </row> + <row> + <entry>Division</entry> + <entry> <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable> </entry> <entry>left</entry> - <entry>Arithmetic multiplication and division.</entry> + <entry>Arithmetic division.</entry> + <entry>6</entry> + </row> + <row> + <entry>Addition</entry> + <entry> + <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable> + </entry> + <entry>left</entry> + <entry>Arithmetic addition.</entry> + <entry>7</entry> </row> <row> + <entry>Subtraction</entry> <entry> - <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>, <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable> </entry> <entry>left</entry> - <entry>Arithmetic addition and subtraction. String or path concatenation (only by <literal>+</literal>).</entry> + <entry>Arithmetic subtraction.</entry> + <entry>7</entry> </row> <row> + <entry>String Concatenation</entry> + <entry> + <replaceable>string1</replaceable> <literal>+</literal> <replaceable>string2</replaceable> + </entry> + <entry>left</entry> + <entry>String concatenation.</entry> + <entry>7</entry> + </row> + <row> + <entry>Not</entry> <entry><literal>!</literal> <replaceable>e</replaceable></entry> <entry>none</entry> <entry>Boolean negation.</entry> + <entry>8</entry> </row> <row> + <entry>Update</entry> <entry><replaceable>e1</replaceable> <literal>//</literal> <replaceable>e2</replaceable></entry> <entry>right</entry> @@ -89,47 +133,90 @@ weakest binding).</para> <replaceable>e2</replaceable> (with the latter taking precedence over the former in case of equally named attributes).</entry> + <entry>9</entry> </row> <row> + <entry>Less Than</entry> <entry> <replaceable>e1</replaceable> <literal><</literal> <replaceable>e2</replaceable>, - <replaceable>e1</replaceable> <literal>></literal> <replaceable>e2</replaceable>, - <replaceable>e1</replaceable> <literal><=</literal> <replaceable>e2</replaceable>, + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Less Than or Equal To</entry> + <entry> + <replaceable>e1</replaceable> <literal><=</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Greater Than</entry> + <entry> + <replaceable>e1</replaceable> <literal>></literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Greater Than or Equal To</entry> + <entry> <replaceable>e1</replaceable> <literal>>=</literal> <replaceable>e2</replaceable> </entry> <entry>none</entry> <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Equality</entry> + <entry> + <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Equality.</entry> + <entry>11</entry> </row> <row> + <entry>Inequality</entry> <entry> - <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>, <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable> </entry> <entry>none</entry> - <entry>Equality and inequality.</entry> + <entry>Inequality.</entry> + <entry>11</entry> </row> <row> + <entry>Logical AND</entry> <entry><replaceable>e1</replaceable> <literal>&&</literal> <replaceable>e2</replaceable></entry> <entry>left</entry> <entry>Logical AND.</entry> + <entry>12</entry> </row> <row> + <entry>Logical OR</entry> <entry><replaceable>e1</replaceable> <literal>||</literal> <replaceable>e2</replaceable></entry> <entry>left</entry> <entry>Logical OR.</entry> + <entry>13</entry> </row> <row> + <entry>Logical Implication</entry> <entry><replaceable>e1</replaceable> <literal>-></literal> <replaceable>e2</replaceable></entry> <entry>none</entry> <entry>Logical implication (equivalent to <literal>!<replaceable>e1</replaceable> || <replaceable>e2</replaceable></literal>).</entry> + <entry>14</entry> </row> </tbody> </tgroup> </table> -</section>
\ No newline at end of file +</section> diff --git a/doc/manual/expressions/simple-building-testing.xml b/doc/manual/expressions/simple-building-testing.xml index 0348c082b..7326a3e76 100644 --- a/doc/manual/expressions/simple-building-testing.xml +++ b/doc/manual/expressions/simple-building-testing.xml @@ -43,7 +43,7 @@ use <command>nix-build</command>’s <option linkend='opt-out-link'>-o</option> switch to give the symlink another name.</para> -<para>Nix has a transactional semantics. Once a build finishes +<para>Nix has transactional semantics. Once a build finishes successfully, Nix makes a note of this in its database: it registers that the path denoted by <envar>out</envar> is now <quote>valid</quote>. If you try to build the derivation again, Nix diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index d1ee0bb2e..e2b8fc867 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -67,5 +67,23 @@ $ sudo launchctl kickstart -k system/org.nixos.nix-daemon </screen> </section> +<section xml:id="sec-installer-proxy-settings"> + +<title>Proxy Environment Variables</title> + +<para>The Nix installer has special handling for these proxy-related +environment variables: +<varname>http_proxy</varname>, <varname>https_proxy</varname>, +<varname>ftp_proxy</varname>, <varname>no_proxy</varname>, +<varname>HTTP_PROXY</varname>, <varname>HTTPS_PROXY</varname>, +<varname>FTP_PROXY</varname>, <varname>NO_PROXY</varname>. +</para> +<para>If any of these variables are set when running the Nix installer, +then the installer will create an override file at +<filename>/etc/systemd/system/nix-daemon.service.d/override.conf</filename> +so <command>nix-daemon</command> will use them. +</para> +</section> + </section> </chapter> diff --git a/doc/manual/installation/prerequisites-source.xml b/doc/manual/installation/prerequisites-source.xml index e87d0de21..e7bdcf966 100644 --- a/doc/manual/installation/prerequisites-source.xml +++ b/doc/manual/installation/prerequisites-source.xml @@ -13,7 +13,7 @@ <listitem><para>Bash Shell. The <literal>./configure</literal> script relies on bashisms, so Bash is required.</para></listitem> - <listitem><para>A version of GCC or Clang that supports C++14.</para></listitem> + <listitem><para>A version of GCC or Clang that supports C++17.</para></listitem> <listitem><para><command>pkg-config</command> to locate dependencies. If your distribution does not provide it, you can get @@ -62,6 +62,10 @@ 1.66.0 or higher. It can be obtained from the official web site <link xlink:href="https://www.boost.org/" />.</para></listitem> + <listitem><para>The <literal>editline</literal> library of version + 1.14.0 or higher. It can be obtained from the its repository + <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem> + <listitem><para>The <command>xmllint</command> and <command>xsltproc</command> programs to build this manual and the man-pages. These are part of the <literal>libxml2</literal> and diff --git a/doc/manual/packages/garbage-collection.xml b/doc/manual/packages/garbage-collection.xml index a1b0ef22a..b506f22b0 100644 --- a/doc/manual/packages/garbage-collection.xml +++ b/doc/manual/packages/garbage-collection.xml @@ -52,12 +52,13 @@ garbage collector as follows: <screen> $ nix-store --gc</screen> -The behaviour of the gargage collector is affected by the <literal>keep- -derivations</literal> (default: true) and <literal>keep-outputs</literal> +The behaviour of the gargage collector is affected by the +<literal>keep-derivations</literal> (default: true) and <literal>keep-outputs</literal> (default: false) options in the Nix configuration file. The defaults will ensure -that all derivations that are not build-time dependencies of garbage collector roots -will be collected but that all output paths that are not runtime dependencies -will be collected. (This is usually what you want, but while you are developing +that all derivations that are build-time dependencies of garbage collector roots +will be kept and that all output paths that are runtime dependencies +will be kept as well. All other derivations or paths will be collected. +(This is usually what you want, but while you are developing it may make sense to keep outputs to ensure that rebuild times are quick.) If you are feeling uncertain, you can also first view what files would diff --git a/doc/manual/packages/s3-substituter.xml b/doc/manual/packages/s3-substituter.xml index e7589ffdb..868b5a66d 100644 --- a/doc/manual/packages/s3-substituter.xml +++ b/doc/manual/packages/s3-substituter.xml @@ -113,7 +113,7 @@ the S3 URL:</para> exactly <uri>s3://example-nix-cache</uri>.</para> <para>Nix will use the <link - xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain</link> for authenticating requests to Amazon S3.</para> @@ -138,7 +138,7 @@ the S3 URL:</para> be <uri>s3://example-nix-cache</uri>.</para> <para>Nix will use the <link - xlink:href="https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html#credentials-default.">default + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default credential provider chain</link> for authenticating requests to Amazon S3.</para> @@ -159,7 +159,6 @@ the S3 URL:</para> "s3:ListBucket", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts", - "s3:ListObjects", "s3:PutObject" ], "Resource": [ diff --git a/doc/manual/release-notes/release-notes.xml b/doc/manual/release-notes/release-notes.xml index e8ff586fa..2655d68e3 100644 --- a/doc/manual/release-notes/release-notes.xml +++ b/doc/manual/release-notes/release-notes.xml @@ -12,6 +12,7 @@ </partintro> --> +<xi:include href="rl-2.3.xml" /> <xi:include href="rl-2.2.xml" /> <xi:include href="rl-2.1.xml" /> <xi:include href="rl-2.0.xml" /> diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 000000000..80fbd6396 --- /dev/null +++ b/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,92 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.3"> + +<title>Release 2.3 (2019-09-04)</title> + +<para>This is primarily a bug fix release. However, it makes some +incompatible changes:</para> + +<itemizedlist> + + <listitem> + <para>Nix now uses BSD file locks instead of POSIX file + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store.</para> + </listitem> + +</itemizedlist> + +<para>It also has the following changes:</para> + +<itemizedlist> + + <listitem> + <para><function>builtins.fetchGit</function>'s <varname>ref</varname> + argument now allows specifying an absolute remote ref. + Nix will automatically prefix <varname>ref</varname> with + <literal>refs/heads</literal> only if <varname>ref</varname> doesn't + already begin with <literal>refs/</literal>. + </para> + </listitem> + + <listitem> + <para>The installer now enables sandboxing by default on + Linux. The <literal>max-jobs</literal> setting now defaults to + 1.</para> + </listitem> + + <listitem> + <para>New builtin functions: + <literal>builtins.isPath</literal>, + <literal>builtins.hashFile</literal>. + </para> + </listitem> + + <listitem> + <para>The <command>nix</command> command has a new + <option>--print-build-logs</option> (<option>-L</option>) flag to + print build log output to stderr, rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + </para> + </listitem> + + <listitem> + <para>Builds are now executed in a pseudo-terminal, and the + <envar>TERM</envar> environment variable is set to + <literal>xterm-256color</literal>. This allows many programs + (e.g. <command>gcc</command>, <command>clang</command>, + <command>cmake</command>) to print colorized log output.</para> + </listitem> + + <listitem> + <para>Add <option>--no-net</option> convenience flag. This flag + disables substituters; sets the <literal>tarball-ttl</literal> + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces.</para> + </listitem> + + <listitem> + <para>Add a <literal>post-build-hook</literal> setting to run a + program after a build has succeeded.</para> + </listitem> + + <listitem> + <para>Add a <literal>trace-function-calls</literal> setting to log + the duration of Nix function calls to stderr.</para> + </listitem> + + <listitem> + <para>On Linux, sandboxing is now disabled by default on systems + that don’t have the necessary kernel support.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/m4/ax_cxx_compile_stdcxx.m4 b/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 000000000..43087b2e6 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,951 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik <bkoz@redhat.com> +# Copyright (c) 2012 Zack Weinberg <zackw@panix.com> +# Copyright (c) 2013 Roy Stogner <roystgnr@ices.utexas.edu> +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov <sokolov@google.com> +# Copyright (c) 2015 Paul Norman <penorman@mac.com> +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016, 2018 Krzesimir Nowak <qdlacz@gmail.com> +# Copyright (c) 2019 Enji Cooper <yaneurabeya@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template <typename T> + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check<void> single_type; + typedef check<check<void>> double_type; + typedef check<check<check<void>>> triple_type; + typedef check<check<check<check<void>>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same<int, decltype(0)>::value == true, ""); + static_assert(is_same<int, decltype(c)>::value == false, ""); + static_assert(is_same<int, decltype(v)>::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same<int, decltype(ac)>::value == true, ""); + static_assert(is_same<int, decltype(av)>::value == true, ""); + static_assert(is_same<int, decltype(sumi)>::value == true, ""); + static_assert(is_same<int, decltype(sumf)>::value == false, ""); + static_assert(is_same<int, decltype(add(c, v))>::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template <int...> + struct sum; + + template <int N0, int... N1toN> + struct sum<N0, N1toN...> + { + static constexpr auto value = N0 + sum<N1toN...>::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template<typename T> + using member = typename T::member_type; + + template<typename T> + void func(...) {} + + template<typename T> + void func(member<T>*) {} + + void test(); + + void test() { func<foo>(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same<T, T> + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same<int, decltype(f(x))>::value, ""); + static_assert(is_same<int&, decltype(g(x))>::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include <initializer_list> +#include <utility> +#include <type_traits> + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template<typename... Args> + int multiply(Args... args) + { + return (args * ... * 1); + } + + template<typename... Args> + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same<std::initializer_list<int>, decltype(foo)>::value); + static_assert(std::is_same<int, decltype(bar)>::value); + } + + namespace test_typename_in_template_template_parameter + { + + template<template<typename> typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template <bool cond> + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template <typename T1, typename T2> + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template <auto n> + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair<int, int> pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair<int, int>& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template<typename T> + Bad + f(T*, T*); + + template<typename T1, typename T2> + Good + f(T1*, T2*); + + static_assert (std::is_same_v<Good, decltype(f(g1, g2))>); + + } + + namespace test_inline_variables + { + + template<class T> void f(T) + {} + + template<class T> inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/m4/ax_cxx_compile_stdcxx_17.m4 b/m4/ax_cxx_compile_stdcxx_17.m4 new file mode 100644 index 000000000..a68341717 --- /dev/null +++ b/m4/ax_cxx_compile_stdcxx_17.m4 @@ -0,0 +1,35 @@ +# ============================================================================= +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html +# ============================================================================= +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++17 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++17. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2015 Moritz Klammler <moritz@klammler.eu> +# Copyright (c) 2016 Krzesimir Nowak <qdlacz@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])]) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 1cdf5ed16..77534babb 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -44,7 +44,7 @@ print STDERR "Nix revision is $nixRev, version is $version\n"; File::Path::make_path($releasesDir); if (system("mountpoint -q $releasesDir") != 0) { - system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; + system("sshfs hydra-mirror\@nixos.org:/releases $releasesDir") == 0 or die; } my $releaseDir = "$releasesDir/nix/$releaseName"; diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index 92ed12fa5..9f26296a9 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -13,8 +13,12 @@ <true/> <key>RunAtLoad</key> <true/> - <key>Program</key> - <string>@bindir@/nix-daemon</string> + <key>ProgramArguments</key> + <array> + <string>/bin/sh</string> + <string>-c</string> + <string>/bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon</string> + </array> <key>StandardErrorPath</key> <string>/var/log/nix-daemon.log</string> <key>StandardOutPath</key> diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 5fc04a3f5..25655204d 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -7,3 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/mk/libraries.mk b/mk/libraries.mk index 3953446cb..307e29b9d 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -91,7 +91,7 @@ define build-library $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) ifneq ($(OS), Darwin) $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) @@ -105,7 +105,7 @@ define build-library $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ -shared $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) ifneq ($(OS), Darwin) diff --git a/mk/programs.mk b/mk/programs.mk index 2fbda12bd..d93df4468 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -32,7 +32,7 @@ define build-program $$(eval $$(call create-dir, $$(_d))) $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $(1)_INSTALL_DIR ?= $$(bindir) $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) @@ -46,7 +46,7 @@ define build-program _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ - $$(trace-ld) $(CXX) -o $$@ $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) else diff --git a/nix.spec.in b/nix.spec.in index 477768c6a..6b9e37637 100644 --- a/nix.spec.in +++ b/nix.spec.in @@ -106,7 +106,7 @@ chmod 1775 $RPM_BUILD_ROOT/nix/store for d in profiles gcroots; do mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user - chmod 1777 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user + chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user done # fix permission of nix profile diff --git a/perl/Makefile b/perl/Makefile index 284c75022..f36f5d0e9 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -4,11 +4,4 @@ GLOBAL_CXXFLAGS += -g -Wall -include Makefile.config -OPTIMIZE = 1 - -ifeq ($(OPTIMIZE), 1) - GLOBAL_CFLAGS += -O3 - GLOBAL_CXXFLAGS += -O3 -endif - include mk/lib.mk diff --git a/perl/configure.ac b/perl/configure.ac index 966700695..e8e3610a8 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -2,8 +2,10 @@ AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX AC_CONFIG_SRCDIR(MANIFEST) AC_CONFIG_AUX_DIR(../config) -CFLAGS= -CXXFLAGS= +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} AC_PROG_CC AC_PROG_CXX AX_CXX_COMPILE_STDCXX_11 diff --git a/release-common.nix b/release-common.nix index 4c5565985..2e8a951b9 100644 --- a/release-common.nix +++ b/release-common.nix @@ -42,7 +42,7 @@ rec { libxml2 libxslt docbook5 - docbook5_xsl + docbook_xsl_ns autoconf-archive autoreconfHook ]; diff --git a/release.nix b/release.nix index ab13451ff..0fe683518 100644 --- a/release.nix +++ b/release.nix @@ -1,5 +1,5 @@ { nix ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; } +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz , officialRelease ? false , systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] }: @@ -72,7 +72,12 @@ let # https://github.com/NixOS/nixpkgs/issues/45462 '' mkdir -p $out/lib - cp ${boost}/lib/libboost_context* $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} ''; configureFlags = configureFlags ++ @@ -165,10 +170,10 @@ let chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user dir=nix-${version}-${system} - fn=$out/$dir.tar.bz2 + fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfj $fn \ + tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --absolute-names \ --hard-dereference \ @@ -295,7 +300,7 @@ let substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ") + (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ") [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${build.x86_64-linux.src.version} diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 87c4c2b05..49076bd5c 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -39,7 +39,7 @@ EOF poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon as a LaunchDaemon" \ - ln -sfn "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" + cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index c89e18b51..13762cba3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -19,9 +19,6 @@ readonly BLUE_UL='\033[38;4;34m' readonly GREEN='\033[38;32m' readonly GREEN_UL='\033[38;4;32m' readonly RED='\033[38;31m' -readonly RED_UL='\033[38;4;31m' -readonly YELLOW='\033[38;33m' -readonly YELLOW_UL='\033[38;4;33m' readonly NIX_USER_COUNT="32" readonly NIX_BUILD_GROUP_ID="30000" @@ -278,73 +275,9 @@ EOF fi if type nix-env 2> /dev/null >&2; then - failure <<EOF -Nix already appears to be installed, and this tool assumes it is -_not_ yet installed. - -$(uninstall_directions) -EOF - fi - - if [ "${NIX_REMOTE:-}" != "" ]; then - failure <<EOF -For some reason, \$NIX_REMOTE is set. It really should not be set -before this installer runs, and it hints that Nix is currently -installed. Please delete the old Nix installation and start again. - -Note: You might need to close your shell window and open a new shell -to clear the variable. -EOF - fi - - if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then - failure <<EOF -It looks like \$SSL_CERT_FILE is set to a path that used to be part of -the old Nix installation. Please unset that variable and try again: - - $ unset SSL_CERT_FILE - -EOF - fi - - for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do - if [ -f "$file" ]; then - if grep -l "^[^#].*.nix-profile" "$file"; then - failure <<EOF -I found a reference to a ".nix-profile" in $file. -This has a high chance of breaking a new nix installation. It was most -likely put there by a previous Nix installer. - -Please remove this reference and try running this again. You should -also look for similar references in: - - - ~/.bash_profile - - ~/.bash_login - - ~/.profile - -or other shell init files that you may have. - -$(uninstall_directions) -EOF - fi - fi - done - - if [ -d /nix ]; then - failure <<EOF -There are some relics of a previous installation of Nix at /nix, and -this scripts assumes Nix is _not_ yet installed. Please delete the old -Nix installation and start again. - -$(uninstall_directions) -EOF - fi - - if [ -d /etc/nix ]; then - failure <<EOF -There are some relics of a previous installation of Nix at /etc/nix, and -this scripts assumes Nix is _not_ yet installed. Please delete the old -Nix installation and start again. + warning <<EOF +Nix already appears to be installed. This installer may run into issues. +If an error occurs, try manually uninstalling, then rerunning this script. $(uninstall_directions) EOF @@ -352,7 +285,7 @@ EOF for profile_target in "${PROFILE_TARGETS[@]}"; do if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then - failure <<EOF + failure <<EOF When this script runs, it backs up the current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though. @@ -364,38 +297,10 @@ in case. 2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like it has anything nix-related in it. If it does, something is probably quite wrong. Please open an issue or get in touch immediately. - -3. Take care to make sure that $profile_target doesn't look like it has -anything nix-related in it. If it does, and $profile_target _did not_, -run: - - $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target - -and try again. -EOF - fi - - if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then - failure <<EOF -It looks like $profile_target already has some Nix configuration in -there. There should be no reason to run this again. If you're having -trouble, please open an issue. EOF fi done - danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile") - for danger_path in "${danger_paths[@]}"; do - if _sudo "making sure that $danger_path doesn't exist" \ - test -e "$danger_path"; then - failure <<EOF -I found a file at $danger_path, which is a relic of a previous -installation. You must first delete this file before continuing. - -$(uninstall_directions) -EOF - fi - done } setup_report() { @@ -529,24 +434,17 @@ create_build_users() { } create_directories() { + # FIXME: remove all of this because it duplicates LocalStore::LocalStore(). + _sudo "to make the basic directory structure of Nix (part 1)" \ - mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} + mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user _sudo "to make the basic directory structure of Nix (part 2)" \ - mkdir -pv -m 1777 /nix/var/nix/{gcroots,profiles}/per-user - - _sudo "to make the basic directory structure of Nix (part 3)" \ mkdir -pv -m 1775 /nix/store - _sudo "to make the basic directory structure of Nix (part 4)" \ + _sudo "to make the basic directory structure of Nix (part 3)" \ chgrp "$NIX_BUILD_GROUP_NAME" /nix/store - _sudo "to set up the root user's profile (part 1)" \ - mkdir -pv -m 0755 /nix/var/nix/profiles/per-user/root - - _sudo "to set up the root user's profile (part 2)" \ - mkdir -pv -m 0700 "$ROOT_HOME/.nix-defexpr" - _sudo "to place the default nix daemon configuration (part 1)" \ mkdir -pv -m 0555 /etc/nix } @@ -589,7 +487,7 @@ EOF We will: - make sure your computer doesn't already have Nix files - (if it does, I will tell you how to clean them up.) + (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users we'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - install Nix in to $NIX_ROOT @@ -749,9 +647,6 @@ setup_default_profile() { place_nix_configuration() { cat <<EOF > "$SCRATCH/nix.conf" build-users-group = $NIX_BUILD_GROUP_NAME - -max-jobs = $NIX_USER_COUNT -cores = 1 EOF _sudo "to place the default nix daemon configuration (part 2)" \ install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf @@ -761,9 +656,13 @@ main() { if [ "$(uname -s)" = "Darwin" ]; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then - # shellcheck source=./install-systemd-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi else failure "Sorry, I don't know what to do on $(uname)" fi @@ -771,9 +670,7 @@ main() { welcome_to_nix chat_about_sudo - if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then - validate_starting_assumptions - fi + validate_starting_assumptions setup_report diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index fc999d336..3f1581854 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -12,7 +12,7 @@ if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 fi -if [ -z "$USER" ]; then +if [ -z "$USER" ] && ! USER=$(id -u -n); then echo "$0: \$USER is not set" >&2 exit 1 fi @@ -141,11 +141,9 @@ if [ -z "$_NIX_INSTALLER_TEST" ]; then fi added= +p=$HOME/.nix-profile/etc/profile.d/nix.sh if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then - # Make the shell source nix.sh during login. - p=$HOME/.nix-profile/etc/profile.d/nix.sh - for i in .bash_profile .bash_login .profile; do fn="$HOME/$i" if [ -w "$fn" ]; then @@ -157,7 +155,6 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then break fi done - fi if [ -z "$added" ]; then diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 04bc539a1..bef3ac4f9 100644..100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -9,6 +9,38 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + +# Path for the systemd override unit file to contain the proxy settings +readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf + +create_systemd_override() { + header "Configuring proxy for the nix-daemon service" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE" +[Service] +$1 +EOF +} + +# Gather all non-empty proxy environment variables into a string +create_systemd_proxy_env() { + vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + for v in $vars; do + if [ "x${!v:-}" != "x" ]; then + echo "Environment=${v}=${!v}" + fi + done +} + +handle_network_proxy() { + # Create a systemd unit override with proxy environment variables + # if any proxy environment variables are not empty. + PROXY_ENV_STRING=$(create_systemd_proxy_env) + if [ -n "${PROXY_ENV_STRING}" ]; then + create_systemd_override "${PROXY_ENV_STRING}" + fi +} + poly_validate_assumptions() { if [ "$(uname -s)" != "Linux" ]; then failure "This script is for use with Linux!" @@ -47,6 +79,8 @@ poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon socket service" \ systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC" + handle_network_proxy + _sudo "to load the systemd unit for nix-daemon" \ systemctl daemon-reload diff --git a/scripts/install.in b/scripts/install.in index 7bff7b216..902758b13 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -18,7 +18,7 @@ cleanup() { trap cleanup EXIT INT QUIT TERM require_util() { - type "$1" > /dev/null 2>&1 || command -v "$1" > /dev/null 2>&1 || + command -v "$1" > /dev/null 2>&1 || oops "you do not have '$1' installed, which I need to $2" } @@ -30,22 +30,21 @@ case "$(uname -s).$(uname -m)" in *) oops "sorry, there is no binary distribution of Nix for your platform";; esac -url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2" +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" -tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")" +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" -require_util bzcat "decompress the binary tarball" require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." curl -L "$url" -o "$tarball" || oops "failed to download '$url'" -if type sha256sum > /dev/null 2>&1; then +if command -v sha256sum > /dev/null 2>&1; then hash2="$(sha256sum -b "$tarball" | cut -c1-64)" -elif type shasum > /dev/null 2>&1; then +elif command -v shasum > /dev/null 2>&1; then hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" -elif type openssl > /dev/null 2>&1; then +elif command -v openssl > /dev/null 2>&1; then hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" else oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" @@ -57,7 +56,7 @@ fi unpack=$tmpDir/unpack mkdir -p "$unpack" -< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'" +tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" script=$(echo "$unpack"/*/install) diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 6940969cc..47655080a 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -2,54 +2,9 @@ if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 -# Set up secure multi-user builds: non-root users build through the -# Nix daemon. -if [ "$USER" != root -o ! -w @localstatedir@/nix/db ]; then - export NIX_REMOTE=daemon -fi - export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" -# Set up the per-user profile. -mkdir -m 0755 -p $NIX_USER_PROFILE_DIR -if ! test -O "$NIX_USER_PROFILE_DIR"; then - echo "WARNING: bad ownership on $NIX_USER_PROFILE_DIR" >&2 -fi - -if test -w $HOME; then - if ! test -L $HOME/.nix-profile; then - if test "$USER" != root; then - ln -s $NIX_USER_PROFILE_DIR/profile $HOME/.nix-profile - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default $HOME/.nix-profile - fi - fi - - # Subscribe the root user to the NixOS channel by default. - if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels - fi - - # Create the per-user garbage collector roots directory. - NIX_USER_GCROOTS_DIR=@localstatedir@/nix/gcroots/per-user/$USER - mkdir -m 0755 -p $NIX_USER_GCROOTS_DIR - if ! test -O "$NIX_USER_GCROOTS_DIR"; then - echo "WARNING: bad ownership on $NIX_USER_GCROOTS_DIR" >&2 - fi - - # Set up a default Nix expression from which to install stuff. - if [ ! -e $HOME/.nix-defexpr -o -L $HOME/.nix-defexpr ]; then - rm -f $HOME/.nix-defexpr - mkdir -p $HOME/.nix-defexpr - if [ "$USER" != root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels $HOME/.nix-defexpr/channels_root - fi - fi -fi - - # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index f3cfa157c..e15f7cd46 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,6 +1,4 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then - __savedpath="$PATH" - export PATH=@coreutils@ # Set up the per-user profile. # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix @@ -9,55 +7,13 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER - mkdir -m 0755 -p "$NIX_USER_PROFILE_DIR" - - if [ "$(stat --printf '%u' "$NIX_USER_PROFILE_DIR")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on "$NIX_USER_PROFILE_DIR", should be $(id -u)" >&2 - fi - - if [ -w "$HOME" ]; then - if ! [ -L "$NIX_LINK" ]; then - echo "Nix: creating $NIX_LINK" >&2 - if [ "$USER" != root ]; then - if ! ln -s "$NIX_USER_PROFILE_DIR"/profile "$NIX_LINK"; then - echo "Nix: WARNING: could not create $NIX_LINK -> $NIX_USER_PROFILE_DIR/profile" >&2 - fi - else - # Root installs in the system-wide profile by default. - ln -s @localstatedir@/nix/profiles/default "$NIX_LINK" - fi - fi - - # Subscribe the user to the unstable Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" - fi - - # Create the per-user garbage collector roots directory. - __user_gcroots=@localstatedir@/nix/gcroots/per-user/"$USER" - mkdir -m 0755 -p "$__user_gcroots" - if [ "$(stat --printf '%u' "$__user_gcroots")" != "$(id -u)" ]; then - echo "Nix: WARNING: bad ownership on $__user_gcroots, should be $(id -u)" >&2 - fi - unset __user_gcroots - - # Set up a default Nix expression from which to install stuff. - __nix_defexpr="$HOME"/.nix-defexpr - [ -L "$__nix_defexpr" ] && rm -f "$__nix_defexpr" - mkdir -m 0755 -p "$__nix_defexpr" - if [ "$USER" != root ] && [ ! -L "$__nix_defexpr"/channels_root ]; then - ln -s @localstatedir@/nix/profiles/per-user/root/channels "$__nix_defexpr"/channels_root - fi - unset __nix_defexpr - fi - # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs> # paths work when the user has fetched the Nixpkgs channel. export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels # Set up environment. # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix - NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_USER_PROFILE_DIR" + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch @@ -78,6 +34,6 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then export MANPATH="$NIX_LINK/share/man:$MANPATH" fi - export PATH="$NIX_LINK/bin:$__savedpath" - unset __savedpath NIX_LINK NIX_USER_PROFILE_DIR NIX_PROFILES + export PATH="$NIX_LINK/bin:$PATH" + unset NIX_LINK NIX_USER_PROFILE_DIR fi @@ -1,6 +1,6 @@ { useClang ? false }: -with import (builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03"; }) {}; +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; with import ./release-common.nix { inherit pkgs; }; diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h index 07de010b1..5a00da3b4 100644 --- a/src/cpptoml/cpptoml.h +++ b/src/cpptoml/cpptoml.h @@ -4,8 +4,8 @@ * @date May 2013 */ -#ifndef _CPPTOML_H_ -#define _CPPTOML_H_ +#ifndef CPPTOML_H +#define CPPTOML_H #include <algorithm> #include <cassert> @@ -84,11 +84,12 @@ class option return &value_; } - const T& value_or(const T& alternative) const + template <class U> + T value_or(U&& alternative) const { if (!empty_) return value_; - return alternative; + return static_cast<T>(std::forward<U>(alternative)); } private: @@ -295,13 +296,12 @@ struct valid_value_or_string_convertible }; template <class T> -struct value_traits<T, typename std:: - enable_if<valid_value_or_string_convertible<T>:: - value>::type> +struct value_traits<T, typename std::enable_if< + valid_value_or_string_convertible<T>::value>::type> { - using value_type = typename std:: - conditional<valid_value<typename std::decay<T>::type>::value, - typename std::decay<T>::type, std::string>::type; + using value_type = typename std::conditional< + valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; using type = value<value_type>; @@ -312,12 +312,11 @@ struct value_traits<T, typename std:: }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_floating_point< - typename std::decay<T>::type>::value>:: - type> +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_floating_point<typename std::decay<T>::type>::value>::type> { using value_type = typename std::decay<T>::type; @@ -330,11 +329,11 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_signed<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && !std::is_floating_point<typename std::decay<T>::type>::value + && std::is_signed<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -356,11 +355,10 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_unsigned<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -395,10 +393,15 @@ struct array_of_trait<array> template <class T> inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); inline std::shared_ptr<array> make_array(); + +namespace detail +{ template <class T> inline std::shared_ptr<T> make_element(); +} + inline std::shared_ptr<table> make_table(); -inline std::shared_ptr<table_array> make_table_array(); +inline std::shared_ptr<table_array> make_table_array(bool is_inline = false); #if defined(CPPTOML_NO_RTTI) /// Base type used to store underlying data type explicitly if RTTI is disabled @@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base> #if defined(CPPTOML_NO_RTTI) base_type type() const { - return type_; + return type_; } protected: @@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as() if (type() == base_type::INT) { auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); - return make_value<double>(static_cast<double>(v->get()));; + return make_value<double>(static_cast<double>(v->get())); } #else if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) @@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const { #if defined(CPPTOML_NO_RTTI) if (type() == base_type::FLOAT) - return std::static_pointer_cast<const value<double>>(shared_from_this()); + return std::static_pointer_cast<const value<double>>( + shared_from_this()); if (type() == base_type::INT) { @@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<array> make_element<array>() { return make_array(); } +} // namespace detail /** * Obtains a option<vector<T>>. The option will be empty if the array @@ -1060,7 +1067,7 @@ class table; class table_array : public base { friend class table; - friend std::shared_ptr<table_array> make_table_array(); + friend std::shared_ptr<table_array> make_table_array(bool); public: std::shared_ptr<base> clone() const override; @@ -1152,14 +1159,25 @@ class table_array : public base array_.reserve(n); } + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + private: #if defined(CPPTOML_NO_RTTI) - table_array() : base(base_type::TABLE_ARRAY) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) { // nothing } #else - table_array() + table_array(bool is_inline = false) : is_inline_(is_inline) { // nothing } @@ -1169,26 +1187,30 @@ class table_array : public base table_array& operator=(const table_array& rhs) = delete; std::vector<std::shared_ptr<table>> array_; + const bool is_inline_ = false; }; -inline std::shared_ptr<table_array> make_table_array() +inline std::shared_ptr<table_array> make_table_array(bool is_inline) { struct make_shared_enabler : public table_array { - make_shared_enabler() + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) { // nothing } }; - return std::make_shared<make_shared_enabler>(); + return std::make_shared<make_shared_enabler>(is_inline); } +namespace detail +{ template <> inline std::shared_ptr<table_array> make_element<table_array>() { - return make_table_array(); + return make_table_array(true); } +} // namespace detail // The below are overloads for fetching specific value types out of a value // where special casting behavior (like bounds checking) is desired @@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<table> make_element<table>() { return make_table(); } +} // namespace detail template <class T> std::shared_ptr<base> value<T>::clone() const @@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const inline std::shared_ptr<base> table_array::clone() const { - auto result = make_table_array(); + auto result = make_table_array(is_inline()); result->reserve(array_.size()); for (const auto& ptr : array_) result->array_.push_back(ptr->clone()->as_table()); @@ -1738,6 +1763,11 @@ inline bool is_number(char c) return c >= '0' && c <= '9'; } +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + /** * Helper object for consuming expected characters. */ @@ -1766,6 +1796,13 @@ class consumer [&](char c) { (*this)(c); }); } + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + int eat_digits(int len) { int val = 0; @@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line) line.push_back(static_cast<char>(c)); } } -} +} // namespace detail /** * The parser class. @@ -1914,21 +1951,25 @@ class parser std::string full_table_name; bool inserted = false; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table name"); if (!full_table_name.empty()) - full_table_name += "."; + full_table_name += '.'; full_table_name += part; if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif if (b->is_table()) curr_table = static_cast<table*>(b.get()); else if (b->is_table_array()) @@ -1946,16 +1987,23 @@ class parser curr_table->insert(part, make_table()); curr_table = static_cast<table*>(curr_table->get(part).get()); } - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); if (it == end) throw_parse_exception( "Unterminated table declaration; did you forget a ']'?"); + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + // table already existed if (!inserted) { @@ -1969,8 +2017,9 @@ class parser // since it has already been defined. If there aren't any // values, then it was implicitly created by something like // [a.b] - if (curr_table->empty() || std::any_of(curr_table->begin(), - curr_table->end(), is_value)) + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) { throw_parse_exception("Redefinition of table " + full_table_name); @@ -1989,36 +2038,45 @@ class parser if (it == end || *it == ']') throw_parse_exception("Table array name cannot be empty"); - std::string full_ta_name; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table array name"); if (!full_ta_name.empty()) - full_ta_name += "."; + full_ta_name += '.'; full_ta_name += part; - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif // if this is the end of the table array name, add an - // element to the table array that we just looked up + // element to the table array that we just looked up, + // provided it was not declared inline if (it != end && *it == ']') { if (!b->is_table_array()) + { throw_parse_exception("Key " + full_ta_name + " is not a table array"); + } + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + v->get().push_back(make_table()); curr_table = v->get().back().get(); } @@ -2059,15 +2117,16 @@ class parser = static_cast<table*>(curr_table->get(part).get()); } } - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); // consume the last "]]" - if (it == end) - throw_parse_exception("Unterminated table array name"); - ++it; - if (it == end) + auto eat = make_consumer(it, end, [this]() { throw_parse_exception("Unterminated table array name"); - ++it; + }); + eat(']'); + eat(']'); consume_whitespace(it, end); eol_or_comment(it, end); @@ -2076,7 +2135,35 @@ class parser void parse_key_value(std::string::iterator& it, std::string::iterator& end, table* curr_table) { - auto key = parse_key(it, end, [](char c) { return c == '='; }); + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast<table*>(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + if (curr_table->contains(key)) throw_parse_exception("Key " + key + " already present"); if (it == end || *it != '=') @@ -2087,18 +2174,57 @@ class parser consume_whitespace(it, end); } - template <class Function> - std::string parse_key(std::string::iterator& it, - const std::string::iterator& end, Function&& fun) + template <class KeyEndFinder, class KeyPartHandler> + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) { consume_whitespace(it, end); - if (*it == '"') + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') { - return parse_quoted_key(it, end); + return string_literal(it, end, *it); } else { - auto bke = std::find_if(it, end, std::forward<Function>(fun)); + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); return parse_bare_key(it, bke); } } @@ -2142,12 +2268,6 @@ class parser return key; } - std::string parse_quoted_key(std::string::iterator& it, - const std::string::iterator& end) - { - return string_literal(it, end, '"'); - } - enum class parse_type { STRING = 1, @@ -2193,7 +2313,7 @@ class parser parse_type determine_value_type(const std::string::iterator& it, const std::string::iterator& end) { - if(it == end) + if (it == end) { throw_parse_exception("Failed to parse value type"); } @@ -2209,7 +2329,11 @@ class parser { return *dtype; } - else if (is_number(*it) || *it == '-' || *it == '+') + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) { return determine_number_type(it, end); } @@ -2235,6 +2359,13 @@ class parser auto check_it = it; if (*check_it == '-' || *check_it == '+') ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + while (check_it != end && is_number(*check_it)) ++check_it; if (check_it != end && *check_it == '.') @@ -2283,57 +2414,56 @@ class parser bool consuming = false; std::shared_ptr<value<std::string>> ret; - auto handle_line - = [&](std::string::iterator& local_it, - std::string::iterator& local_end) { - if (consuming) - { - local_it = std::find_if_not(local_it, local_end, is_ws); - - // whole line is whitespace - if (local_it == local_end) - return; - } - - consuming = false; - - while (local_it != local_end) - { - // handle escaped characters - if (delim == '"' && *local_it == '\\') - { - auto check = local_it; - // check if this is an actual escape sequence or a - // whitespace escaping backslash - ++check; - consume_whitespace(check, local_end); - if (check == local_end) - { - consuming = true; - break; - } - - ss << parse_escape_code(local_it, local_end); - continue; - } - - // if we can end the string - if (std::distance(local_it, local_end) >= 3) - { - auto check = local_it; - // check for """ - if (*check++ == delim && *check++ == delim - && *check++ == delim) - { - local_it = check; - ret = make_value<std::string>(ss.str()); - break; - } - } - - ss << *local_it++; - } - }; + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; // handle the remainder of the current line handle_line(it, end); @@ -2514,17 +2644,13 @@ class parser return value; } - bool is_hex(char c) - { - return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - uint32_t hex_to_digit(char c) { if (is_number(c)) return static_cast<uint32_t>(c - '0'); - return 10 + static_cast<uint32_t>( - c - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + return 10 + + static_cast<uint32_t>(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); } std::shared_ptr<base> parse_number(std::string::iterator& it, @@ -2538,17 +2664,23 @@ class parser ++check_it; }; - eat_sign(); + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; - auto eat_numbers = [&]() { + auto eat_digits = [&](bool (*check_char)(char)) { auto beg = check_it; - while (check_it != end && is_number(*check_it)) + while (check_it != end && check_char(*check_it)) { ++check_it; if (check_it != end && *check_it == '_') { ++check_it; - if (check_it == end || !is_number(*check_it)) + if (check_it == end || !check_char(*check_it)) throw_parse_exception("Malformed number"); } } @@ -2557,15 +2689,63 @@ class parser throw_parse_exception("Malformed number"); }; - auto check_no_leading_zero = [&]() { - if (check_it != end && *check_it == '0' && check_it + 1 != check_end - && check_it[1] != '.') + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') { - throw_parse_exception("Numbers may not have leading zeros"); + eat_hex(); + return parse_int(it, check_it, 16); } - }; + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + eat_sign(); check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits<double>::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits<double>::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + eat_numbers(); if (check_it != end @@ -2604,14 +2784,17 @@ class parser } std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, - const std::string::iterator& end) + const std::string::iterator& end, + int base = 10, + const char* prefix = "") { std::string v{it, end}; + v = prefix + v; v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); it = end; try { - return make_value<int64_t>(std::stoll(v)); + return make_value<int64_t>(std::stoll(v, nullptr, base)); } catch (const std::invalid_argument& ex) { @@ -2674,18 +2857,33 @@ class parser std::string::iterator find_end_of_number(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { + auto ret = std::find_if(it, end, [](char c) { return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' - && c != '-' && c != '+'; + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; } std::string::iterator find_end_of_date(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { - return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-' - && c != '+' && c != '.'; + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; }); } @@ -2754,7 +2952,7 @@ class parser if (it == date_end) return make_value(ldate); - eat('T'); + eat.eat_or('T', ' '); local_datetime ldt; static_cast<local_date&>(ldt) = ldate; @@ -2850,9 +3048,9 @@ class parser auto arr = make_array(); while (it != end && *it != ']') { - auto value = parse_value(it, end); - if (auto v = value->as<Value>()) - arr->get().push_back(value); + auto val = parse_value(it, end); + if (auto v = val->as<Value>()) + arr->get().push_back(val); else throw_parse_exception("Arrays must be homogeneous"); skip_whitespace_and_comments(it, end); @@ -2871,7 +3069,7 @@ class parser std::string::iterator& it, std::string::iterator& end) { - auto arr = make_element<Object>(); + auto arr = detail::make_element<Object>(); while (it != end && *it != ']') { @@ -2881,7 +3079,7 @@ class parser arr->get().push_back(((*this).*fun)(it, end)); skip_whitespace_and_comments(it, end); - if (*it != ',') + if (it == end || *it != ',') break; ++it; @@ -2906,8 +3104,11 @@ class parser throw_parse_exception("Unterminated inline table"); consume_whitespace(it, end); - parse_key_value(it, end, tbl.get()); - consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } } while (*it == ','); if (it == end || *it != '}') @@ -2987,7 +3188,8 @@ class parser if (it[4] != '-' || it[7] != '-') return {}; - if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end)) + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) { // datetime type auto time_end = find_end_of_time(it + 11, date_end); @@ -3243,7 +3445,7 @@ class toml_writer { res += "\\\\"; } - else if ((const uint32_t)*it <= 0x001f) + else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f)) { res += "\\u"; std::stringstream ss; @@ -3274,12 +3476,21 @@ class toml_writer */ void write(const value<double>& v) { - std::ios::fmtflags flags{stream_.flags()}; - - stream_ << std::showpoint; - write(v.get()); - - stream_.flags(flags); + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits<double>::max_digits10) + << v.get(); + + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); + + stream_ << double_str; + has_naked_endline_ = false; } /** @@ -3287,9 +3498,9 @@ class toml_writer * offset_datetime. */ template <class T> - typename std::enable_if<is_one_of<T, int64_t, local_date, local_time, - local_datetime, - offset_datetime>::value>::type + typename std::enable_if< + is_one_of<T, int64_t, local_date, local_time, local_datetime, + offset_datetime>::value>::type write(const value<T>& v) { write(v.get()); @@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a) a.accept(writer); return stream; } -} -#endif +} // namespace cpptoml +#endif // CPPTOML_H diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..13950ab8d 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8f728b906..dc16d20b5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "json.hh" #include <algorithm> +#include <chrono> #include <cstring> #include <unistd.h> #include <sys/time.h> @@ -16,7 +17,6 @@ #include <iostream> #include <fstream> -#include <sys/time.h> #include <sys/resource.h> #if HAVE_BOEHMGC @@ -46,11 +46,10 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con { checkInterrupt(); - if (active.find(&v) != active.end()) { + if (!active.insert(&v).second) { str << "<CYCLE>"; return; } - active.insert(&v); switch (v.type) { case tInt: @@ -1094,9 +1093,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + std::optional<FunctionCallTrace> trace; + if (evalSettings.traceFunctionCalls) { + trace.emplace(pos); + } + forceValue(fun, pos); if (fun.type == tPrimOp || fun.type == tPrimOpApp) { @@ -1442,8 +1445,7 @@ void EvalState::forceValueDeep(Value & v) std::function<void(Value & v)> recurse; recurse = [&](Value & v) { - if (seen.find(&v) != seen.end()) return; - seen.insert(&v); + if (!seen.insert(&v).second) return; forceValue(v); @@ -1872,8 +1874,7 @@ size_t valueSize(Value & v) std::set<const void *> seen; auto doString = [&](const char * s) -> size_t { - if (seen.find(s) != seen.end()) return 0; - seen.insert(s); + if (!seen.insert(s).second) return 0; return strlen(s) + 1; }; @@ -1881,8 +1882,7 @@ size_t valueSize(Value & v) std::function<size_t(Env & v)> doEnv; doValue = [&](Value & v) -> size_t { - if (seen.find(&v) != seen.end()) return 0; - seen.insert(&v); + if (!seen.insert(&v).second) return 0; size_t sz = sizeof(Value); @@ -1897,8 +1897,7 @@ size_t valueSize(Value & v) sz += doString(v.path); break; case tAttrs: - if (seen.find(v.attrs) == seen.end()) { - seen.insert(v.attrs); + if (seen.insert(v.attrs).second) { sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); for (auto & i : *v.attrs) sz += doValue(*i.value); @@ -1907,8 +1906,7 @@ size_t valueSize(Value & v) case tList1: case tList2: case tListN: - if (seen.find(v.listElems()) == seen.end()) { - seen.insert(v.listElems()); + if (seen.insert(v.listElems()).second) { sz += v.listSize() * sizeof(Value *); for (size_t n = 0; n < v.listSize(); ++n) sz += doValue(*v.listElems()[n]); @@ -1929,8 +1927,7 @@ size_t valueSize(Value & v) sz += doValue(*v.primOpApp.right); break; case tExternal: - if (seen.find(v.external) != seen.end()) break; - seen.insert(v.external); + if (!seen.insert(v.external).second) break; sz += v.external->valueSize(seen); break; default: @@ -1941,8 +1938,7 @@ size_t valueSize(Value & v) }; doEnv = [&](Env & env) -> size_t { - if (seen.find(&env) != seen.end()) return 0; - seen.insert(&env); + if (!seen.insert(&env).second) return 0; size_t sz = sizeof(Env) + sizeof(Value *) * env.size; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 82e6ca877..61ee4a73b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,6 +6,7 @@ #include "symbol-table.hh" #include "hash.hh" #include "config.hh" +#include "function-trace.hh" #include <map> #include <optional> @@ -353,6 +354,9 @@ struct EvalSettings : Config Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 000000000..8b0ec848d --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "eval.hh" +#include <sys/time.h> + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + + FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); + } + + ~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); + } +}; +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 21a4d7917..4e22a1d77 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -277,8 +277,7 @@ static bool getDerivation(EvalState & state, Value & v, /* Remove spurious duplicates (e.g., a set like `rec { x = derivation {...}; y = x;}'. */ - if (done.find(v.attrs) != done.end()) return false; - done.insert(v.attrs); + if (!done.insert(v.attrs).second) return false; DrvInfo drv(state, attrPath, v.attrs); diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 3f6017957..8bae986f9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) mkFloat(v, stod(tmp_number)); else mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range e) { + } catch (std::out_of_range & e) { throw JSONParseError("out-of-range JSON number"); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 787039307..93834bec6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,5 +1,5 @@ %glr-parser -%pure-parser +%define api.pure %locations %define parse.error verbose %defines @@ -81,6 +81,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); @@ -102,11 +104,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, attrs = nested; } } + // Expr insertion. + // ========================== if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast<ExprAttrs *>(e); + auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } } else { + // This attr path is not defined. Let's create it. attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); e->setName(i->symbol); } @@ -118,11 +138,10 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { - if (formals->argNames.find(formal.name) != formals->argNames.end()) + if (!formals->argNames.insert(formal.name).second) throw ParseError(format("duplicate formal function argument '%1%' at %2%") % formal.name % pos); formals->formals.push_front(formal); - formals->argNames.insert(formal.name); } @@ -657,7 +676,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 06f577f36..deaffa605 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -396,8 +396,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar throw EvalError(format("attribute 'key' required, at %1%") % pos); state.forceValue(*key->value); - if (doneKeys.find(key->value) != doneKeys.end()) continue; - doneKeys.insert(key->value); + if (!doneKeys.insert(key->value).second) continue; res.push_back(e); /* Call the `operator' function with `e' as argument. */ @@ -832,8 +831,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + try { mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { @@ -1267,13 +1272,12 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, string name = state.forceStringNoCtx(*j->value, pos); Symbol sym = state.symbols.create(name); - if (seen.find(sym) == seen.end()) { + if (seen.insert(sym).second) { Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); if (j2 == v2.attrs->end()) throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); - seen.insert(sym); } } @@ -2050,9 +2054,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2063,32 +2067,32 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + auto res = getDownloader()->downloadCached(state.store, request); if (state.allowedPaths) - state.allowedPaths->insert(res); + state.allowedPaths->insert(res.path); - mkString(v, res, PathSet({res})); + mkString(v, res.storePath, PathSet({res.storePath})); } diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index aaf02c856..90f600284 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -38,7 +38,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, try { runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError e) { + } catch (ExecError & e) { if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; clean = false; } @@ -94,7 +94,11 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, runProgram("git", true, { "init", "--bare", cacheDir }); } - Path localRefFile = cacheDir + "/refs/heads/" + *ref; + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; bool doFetch; time_t now = time(0); @@ -116,7 +120,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; } if (doFetch) { @@ -235,7 +239,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 66f49f374..a907d0e1c 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now) + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ @@ -96,17 +96,14 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, try { runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); } - catch (ExecError & e){ + catch (ExecError & e) { string transJournal = cacheDir + "/.hg/store/journal"; /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) - { + if (pathExists(transJournal)) { runProgram("hg", true, { "recover", "-R", cacheDir }); runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - else - { - throw ExecError(e.status, fmt("program hg '%1%' ", statusToString(e.status))); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); } } } else { @@ -214,7 +211,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4128de05d..a84e569e9 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); } + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + else if (t->is_value()) { if (auto val = t->as<int64_t>()) mkInt(v, val->get()); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 00b1918a8..1f0b1541d 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -105,10 +105,9 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "derivation", xmlAttrs); - if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { - drvsSeen.insert(drvPath); + if (drvPath != "" && drvsSeen.insert(drvPath).second) showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); - } else + else doc.writeEmptyElement("repeated"); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 4c35a4199..9e1d7cee6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { + settings.set("max-jobs", s); + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54d..d3dbfbc44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -80,6 +80,7 @@ string getArg(const string & opt, } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif static void sigHandler(int signo) { } @@ -105,9 +107,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif loadConfFile(); @@ -125,6 +129,15 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); @@ -175,10 +188,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description("build from source if substitution fails") .set(&(bool&) settings.tryFallback, true); - mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { - settings.set("max-jobs", s); - }); - auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4527ee6ba..8e6f1f55d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,10 +10,13 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "thread-pool.hh" #include <chrono> - #include <future> +#include <regex> + +#include <nlohmann/json.hpp> namespace nix { @@ -55,7 +58,7 @@ void BinaryCacheStore::init() } void BinaryCacheStore::getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) + Callback<std::shared_ptr<std::string>> callback) noexcept { try { callback(getFile(path)); @@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(info.path, *nar, narAccessor); + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str JSONObject jsonRoot(jsonOut); jsonRoot.attr("version", 1); - auto narAccessor = makeNarAccessor(nar); - - if (accessor_) - accessor_->addToCache(info.path, *nar, narAccessor); - { auto res = jsonRoot.placeholder("root"); listNar(res, narAccessor, "", true); @@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); } - else { - if (accessor_) - accessor_->addToCache(info.path, *nar, makeNarAccessor(nar)); - } - /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); @@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); - /* Atomically write the NAR file. */ narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : ""); + + /* Optionally maintain an index of DWARF debug info files + consisting of JSON files named 'debuginfo/<build-id>' that + specify the NAR file and member containing the debug info. */ + if (writeDebugInfo) { + + std::string buildIdDir = "/lib/debug/.build-id"; + + if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + + ThreadPool threadPool(25); + + auto doFile = [&](std::string member, std::string key, std::string target) { + checkInterrupt(); + + nlohmann::json json; + json["archive"] = target; + json["member"] = member; + + // FIXME: or should we overwrite? The previous link may point + // to a GC'ed file, so overwriting might be useful... + if (fileExists(key)) return; + + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + + upsertFile(key, json.dump(), "application/json"); + }; + + std::regex regex1("^[0-9a-f]{2}$"); + std::regex regex2("^[0-9a-f]{38}\\.debug$"); + + for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + "/" + s1; + + if (narAccessor->stat(dir).type != FSAccessor::tDirectory + || !std::regex_match(s1, regex1)) + continue; + + for (auto & s2 : narAccessor->readDirectory(dir)) { + auto debugPath = dir + "/" + s2; + + if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + || !std::regex_match(s2, regex2)) + continue; + + auto buildId = s1 + s2; + + std::string key = "debuginfo/" + buildId; + std::string target = "../" + narInfo->url; + + threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + } + } + + threadPool.process(); + } + } + + /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); @@ -240,7 +296,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback<std::shared_ptr<ValidPathInfo>> callback) + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { auto uri = getUri(); auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, @@ -249,21 +305,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + getFile(narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) { try { auto data = fut.get(); - if (!data) return callback(nullptr); + if (!data) return (*callbackPtr)(nullptr); stats.narInfoRead++; - callback((std::shared_ptr<ValidPathInfo>) + (*callbackPtr)((std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile)); (void) act; // force Activity into this lambda to ensure it stays alive } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 953f3b90a..c77292294 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,6 +17,7 @@ public: const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<bool> parallelCompression{this, false, "parallel-compression", @@ -47,7 +48,7 @@ public: /* Fetch the specified file and call the specified callback with the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback); + Callback<std::shared_ptr<std::string>> callback) noexcept; std::shared_ptr<std::string> getFile(const std::string & path); @@ -73,7 +74,7 @@ public: bool isValidPathUncached(const Path & path) override; void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0bd738809..68c2f2ce3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -38,6 +38,7 @@ #include <unistd.h> #include <errno.h> #include <cstring> +#include <termios.h> #include <pwd.h> #include <grp.h> @@ -265,6 +266,12 @@ public: /* Set if at least one derivation had a timeout. */ bool timedOut; + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + LocalStore & store; std::unique_ptr<HookInstance> hook; @@ -561,10 +568,9 @@ UserLock::UserLock() { auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(fnUserLock)) + if (!lockedPaths->insert(fnUserLock).second) /* We already have a lock on this one. */ continue; - lockedPaths->insert(fnUserLock); } try { @@ -613,8 +619,8 @@ UserLock::UserLock() UserLock::~UserLock() { auto lockedPaths(lockedPaths_.lock()); - assert(lockedPaths->count(fnUserLock)); - lockedPaths->erase(fnUserLock); + auto erased = lockedPaths->erase(fnUserLock); + assert(erased); } @@ -721,23 +727,6 @@ HookInstance::~HookInstance() ////////////////////////////////////////////////////////////////////// -typedef map<std::string, std::string> StringRewrites; - - -std::string rewriteStrings(std::string s, const StringRewrites & rewrites) -{ - for (auto & i : rewrites) { - size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) - s.replace(j, i.first.size(), i.second); - } - return s; -} - - -////////////////////////////////////////////////////////////////////// - - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; class SubstitutionGoal; @@ -859,7 +848,7 @@ private: #endif /* Hash rewriting. */ - StringRewrites inputRewrites, outputRewrites; + StringMap inputRewrites, outputRewrites; typedef map<Path, Path> RedirectedOutputs; RedirectedOutputs redirectedOutputs; @@ -1118,10 +1107,8 @@ void DerivationGoal::addWantedOutputs(const StringSet & outputs) needRestart = true; } else for (auto & i : outputs) - if (wantedOutputs.find(i) == wantedOutputs.end()) { - wantedOutputs.insert(i); + if (wantedOutputs.insert(i).second) needRestart = true; - } } @@ -1190,7 +1177,7 @@ void DerivationGoal::haveDerivation() /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drv->substitutesAllowed()) + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); @@ -1558,8 +1545,8 @@ void DerivationGoal::buildDone() if (hook) { hook->builderOut.readSide = -1; hook->fromHook.readSide = -1; - } - else builderOut.readSide = -1; + } else + builderOut.readSide = -1; /* Close the log file. */ closeLogFile(); @@ -1622,6 +1609,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map<std::string, std::string> hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; @@ -2181,7 +2223,48 @@ void DerivationGoal::startBuilder() Path logFile = openLogFile(); /* Create a pipe to get the output of the builder. */ - builderOut.create(); + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); result.startTime = time(0); @@ -2254,17 +2337,37 @@ void DerivationGoal::startBuilder() flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) + if (child == -1 && errno == EINVAL) { /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unprivileged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); _exit(0); }, options); - if (helper.wait() != 0) + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + tmpDirInSandbox = tmpDir; + goto fallback; + } else if (res != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2287,14 +2390,14 @@ void DerivationGoal::startBuilder() writeFile("/proc/" + std::to_string(pid) + "/gid_map", (format("%d %d 1") % sandboxGid % hostGid).str()); - /* Signal the builder that we've updated its user - namespace. */ + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; } else #endif { + fallback: options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); @@ -2406,6 +2509,9 @@ void DerivationGoal::initEnv() may change that in the future. So tell the builder which file descriptor to use for that. */ env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; } @@ -2683,7 +2789,13 @@ void DerivationGoal::runChild() on. */ if (fixedOutput) { ss.push_back("/etc/resolv.conf"); - ss.push_back("/etc/nsswitch.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); if (pathExists("/var/run/nscd/socket")) @@ -3168,6 +3280,7 @@ void DerivationGoal::registerOutputs() /* Throw an error after registering the path as valid. */ + worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", dest, h.to_string(), h2.to_string())); @@ -3210,6 +3323,7 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { + worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); @@ -3221,10 +3335,10 @@ void DerivationGoal::registerOutputs() buildUser ? buildUser->getGID() : getgid(), path, dst, drvPath, tmpDir); - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") % drvPath % path); } @@ -3925,17 +4039,6 @@ void SubstitutionGoal::tryToRun() return; } - /* If the store path is already locked (probably by a - DerivationGoal), then put this goal to sleep. Note: we don't - acquire a lock here since that breaks addToStore(), so below we - handle an AlreadyLocked exception from addToStore(). The check - here is just an optimisation to prevent having to redo a - download due to a locked path. */ - if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { - worker.waitForAWhile(shared_from_this()); - return; - } - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3975,12 +4078,6 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (AlreadyLocked & e) { - /* Probably a DerivationGoal is already building this store - path. Sleep for a while and try again. */ - state = &SubstitutionGoal::init; - worker.waitForAWhile(shared_from_this()); - return; } catch (std::exception & e) { printError(e.what()); @@ -4056,6 +4153,8 @@ Worker::Worker(LocalStore & store) lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; + hashMismatch = false; + checkMismatch = false; } @@ -4361,14 +4460,15 @@ void Worker::waitForInput() for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { ssize_t rd = read(k, buffer.data(), buffer.size()); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); @@ -4415,7 +4515,29 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return timedOut ? 101 : (permanentFailure ? 100 : 1); + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 + if (timedOut) + mask |= 0x01; // 101 + if (hashMismatch) + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) + mask |= 0x60; + return mask ? mask : 1; } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 74e706664..096593886 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -123,8 +123,7 @@ static Path out; static void addPkg(const Path & pkgDir, int priority) { - if (done.count(pkgDir)) return; - done.insert(pkgDir); + if (!done.insert(pkgDir).second) return; createLinks(pkgDir, out, priority); try { diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 92aec63a0..b1af3b4fc 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -64,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3961126ff..23fcfb281 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,12 +36,6 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::substitutesAllowed() const -{ - return get(env, "allowSubstitutes", "1") == "1"; -} - - bool BasicDerivation::isBuiltin() const { return string(builder, 0, 8) == "builtin:"; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 9753e796d..8e02c9bc5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,8 +56,6 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool substitutesAllowed() const; - bool isBuiltin() const; /* Return true iff this is a fixed-output derivation. */ diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 22382ab1d..cdf56e09d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -30,23 +30,7 @@ using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting<bool> enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting<size_t> httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); @@ -87,19 +71,24 @@ struct CurlDownloader : public Downloader std::string encoding; + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, - Callback<DownloadResult> callback) + Callback<DownloadResult> && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) - , callback(callback) + , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { - if (this->request.dataCallback) + if (this->request.dataCallback) { + writtenToSink += len; this->request.dataCallback((char *) data, len); - else + } else this->result.data->append((char *) data, len); }) { @@ -177,6 +166,7 @@ struct CurlDownloader : public Downloader status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); result.bodySize = 0; + acceptRanges = false; encoding = ""; } else { auto i = line.find(':'); @@ -194,7 +184,9 @@ struct CurlDownloader : public Downloader return 0; } } else if (name == "content-encoding") - encoding = trim(string(line, i + 1));; + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; } } return realSize; @@ -244,8 +236,6 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } - long lowSpeedTimeout = 300; - void init() { if (!req) req = curl_easy_init(); @@ -270,6 +260,8 @@ struct CurlDownloader : public Downloader #if LIBCURL_VERSION_NUM >= 0x072f00 if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); @@ -303,13 +295,16 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + result.data = std::make_shared<std::string>(); result.bodySize = 0; } @@ -319,16 +314,21 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; @@ -339,18 +339,12 @@ struct CurlDownloader : public Downloader failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); done = true; - - try { - act.progress(result.bodySize, result.bodySize); - callback(std::move(result)); - } catch (...) { - done = true; - callback.rethrow(); - } + callback(std::move(result)); } else { @@ -412,10 +406,20 @@ struct CurlDownloader : public Downloader request.verb(), request.uri, curl_easy_strerror(code), code)); /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient + && attempt < request.tries + && (!this->request.dataCallback + || writtenToSink == 0 + || (acceptRanges && encoding.empty()))) + { int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); downloader.enqueueItem(shared_from_this()); } @@ -588,7 +592,7 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError(format("unexpected error in download thread: %s") % e.what()); + printError("unexpected error in download thread: %s", e.what()); } { @@ -661,7 +665,7 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared<DownloadItem>(*this, request, callback)); + enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); } }; @@ -790,20 +794,26 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref<Store> store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -822,6 +832,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -829,10 +841,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -845,17 +857,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -870,11 +882,12 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("warning: %s; using cached result", e.msg()); + result.etag = expectedETag; } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -897,14 +910,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, expectedHash.to_string(), gotHash.to_string()); + unsigned int statusCode = 102; + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + url, request.expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..68565bf46 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,37 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + + Setting<unsigned int> tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -36,15 +60,39 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr<std::string> data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + +struct CachedDownloadResult +{ + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional<std::string> etag; + std::string effectiveUri; +}; + class Store; struct Downloader { + virtual ~Downloader() { } + /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ @@ -64,8 +112,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f703e1d15..64780a6da 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -19,6 +19,8 @@ public: uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 26e2b0dca..7d3da1cc8 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); @@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } @@ -221,26 +221,22 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); //if (*fd == -1) continue; - if (path != fnTempRoots) { - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + /* Read the entire file. */ string contents = readFile(fd->get()); @@ -444,17 +440,22 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #if !defined(__linux__) - try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - unchecked[match[1]].emplace("{lsof}"); + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ } #endif @@ -689,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state) throw SysError(format("statting '%1%'") % path); if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; continue; } @@ -700,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting '%1%'") % path); - state.results.bytesFreed += st.st_blocks * 512ULL; + state.results.bytesFreed += st.st_size; } struct stat st; @@ -783,7 +783,11 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) assertStorePath(i); tryToDelete(state, i); if (state.dead.find(i) == state.dead.end()) - throw Error(format("cannot delete path '%1%' since it is still alive") % i); + throw Error(format( + "cannot delete path '%1%' since it is still alive. " + "To find out why use: " + "nix-store --query --roots" + ) % i); } } else if (options.maxFreed > 0) { @@ -866,7 +870,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - auto getAvail = [this]() { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); @@ -887,7 +896,7 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; auto avail = getAvail(); @@ -914,11 +923,11 @@ void LocalStore::autoGC(bool sync) promise.set_value(); }); - printInfo("running auto-GC to free %d bytes", settings.maxFree - avail); - GCOptions options; options.maxFreed = settings.maxFree - avail; + printInfo("running auto-GC to free %d bytes", options.maxFreed); + GCResults results; collectGarbage(options, results); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1c2c08715..249c36673 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -105,6 +105,13 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +void Settings::requireExperimentalFeature(const std::string & name) +{ + auto & f = experimentalFeatures.get(); + if (std::find(f.begin(), f.end(), name) == f.end()) + throw Error("experimental Nix feature '%s' is disabled", name); +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting<SandboxMode>::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 53efc6a90..1221e4db7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -209,6 +209,9 @@ public: "The paths to make available inside the build sandbox.", {"build-chroot-dirs", "build-sandbox-paths"}}; + Setting<bool> sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; @@ -255,7 +258,7 @@ public: "Secret keys with which to sign local builds."}; Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting<bool> requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " @@ -315,6 +318,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting<std::string> postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; @@ -342,8 +348,16 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + + Setting<Strings> experimentalFeatures{this, {}, "experimental-features", + "Experimental Nix features to enable."}; + + void requireExperimentalFeature(const std::string & name); }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8da0e2f9d..779f89e68 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -84,7 +84,6 @@ protected: try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { @@ -114,7 +113,6 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } @@ -133,23 +131,25 @@ protected: } void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) override + Callback<std::shared_ptr<std::string>> callback) noexcept override { checkEnabled(); auto request(makeRequest(path)); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + getDownloader()->enqueueDownload(request, - {[callback, this](std::future<DownloadResult> result) { + {[callbackPtr, this](std::future<DownloadResult> result) { try { - callback(result.get().data); + (*callbackPtr)(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr<std::string>()); + return (*callbackPtr)(std::shared_ptr<std::string>()); maybeDisable(); - callback.rethrow(); + callbackPtr->rethrow(); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7c9bc2b68..d5fbdd25a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override { try { auto conn(connections->get()); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b7001795b..9f99ee76d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -63,6 +63,8 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + if (writeDebugInfo) + createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..49061f27d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -5,6 +5,7 @@ #include "worker-protocol.hh" #include "derivations.hh" #include "nar-info.hh" +#include "references.hh" #include <iostream> #include <algorithm> @@ -70,15 +71,17 @@ LocalStore::LocalStore(const Params & params) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + + createUser(getUserName(), getuid()); + /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { - - Path perUserDir = profilesDir + "/per-user"; - createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 01777) == -1) - throw SysError(format("could not set permissions on '%1%' to 1777") % perUserDir); - mode_t perm = 01775; struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -629,7 +632,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { try { auto info = std::make_shared<ValidPathInfo>(); @@ -879,8 +882,8 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, info->references, narInfo ? narInfo->fileSize : 0, info->narSize}; - } catch (InvalidPath) { - } catch (SubstituterDisabled) { + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) printError(e.what()); @@ -1007,17 +1010,24 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + std::unique_ptr<AbstractHashSink> hashSink; + if (info.ca == "") + hashSink = std::make_unique<HashSink>(htSHA256); + else { + if (!info.references.empty()) + settings.requireExperimentalFeature("ca-references"); + hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path)); + } LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); - hashSink(data, n); + (*hashSink)(data, n); return n; }); restorePath(realPath, wrapperSource); - auto hashResult = hashSink.finish(); + auto hashResult = hashSink->finish(); if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1210,7 +1220,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); PathSet store; @@ -1221,13 +1232,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + fdGCLock = -1; + for (auto & i : validPaths2) verifyPath(i, store, done, validPaths, repair, errors); - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock = -1; - /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking hashes..."); @@ -1240,7 +1249,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of '%1%'") % i); - HashResult current = hashPath(info->narHash.type, toRealPath(i)); + + std::unique_ptr<AbstractHashSink> hashSink; + if (info->ca == "") + hashSink = std::make_unique<HashSink>(info->narHash.type); + else + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path)); + + dumpPath(toRealPath(i), *hashSink); + auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError(format("path '%1%' was modified! " @@ -1293,8 +1310,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, { checkInterrupt(); - if (done.find(path) != done.end()) return; - done.insert(path); + if (!done.insert(path).second) return; if (!isStorePath(path)) { printError(format("path '%1%' is not in the Nix store") % path); @@ -1434,4 +1450,19 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } +void LocalStore::createUser(const std::string & userName, uid_t userId) +{ + for (auto & dir : { + fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName) + }) { + createDirs(dir); + if (chmod(dir.c_str(), 0755) == -1) + throw SysError("changing permissions of directory '%s'", dir); + if (chown(dir.c_str(), userId, getgid()) == -1) + throw SysError("changing owner of directory '%s'", dir); + } +} + + } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6b655647b..379a06af8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; @@ -263,7 +263,7 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); @@ -293,6 +293,8 @@ private: Path getRealStoreDir() override { return realStoreDir; } + void createUser(const std::string & userName, uid_t userId) override; + friend class DerivationGoal; friend class SubstitutionGoal; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 89fc918c3..d690fea28 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -39,9 +39,12 @@ libstore_CXXFLAGS = \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_MAN_DIR=\"$(mandir)\" \ - -DSANDBOX_SHELL="\"$(sandbox_shell)\"" \ -DLSOF=\"$(lsof)\" +ifneq ($(sandbox_shell),) +libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" +endif + $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/build.cc: diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index adcce026f..05b93d4c9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "derivations.hh" +#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" #include "store-api.hh" @@ -28,8 +29,7 @@ void Store::computeFSClosure(const PathSet & startPaths, { auto state(state_.lock()); if (state->exc) return; - if (state->paths.count(path)) return; - state->paths.insert(path); + if (!state->paths.insert(path).second) return; state->pending++; } @@ -174,8 +174,7 @@ void Store::queryMissing(const PathSet & targets, { auto state(state_.lock()); - if (state->done.count(path)) return; - state->done.insert(path); + if (!state->done.insert(path).second) return; } DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); @@ -189,6 +188,7 @@ void Store::queryMissing(const PathSet & targets, } Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); PathSet invalid; for (auto & j : drv.outputs) @@ -197,7 +197,7 @@ void Store::queryMissing(const PathSet & targets, invalid.insert(j.second.path); if (invalid.empty()) return; - if (settings.useSubstitutes && drv.substitutesAllowed()) { + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); @@ -250,8 +250,7 @@ Paths Store::topoSortPaths(const PathSet & paths) if (parents.find(path) != parents.end()) throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); - if (visited.find(path) != visited.end()) return; - visited.insert(path); + if (!visited.insert(path).second) return; parents.insert(path); PathSet references; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 17fde00a0..87be8a24e 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -108,4 +108,9 @@ bool ParsedDerivation::willBuildLocally() const return getBoolAttr("preferLocalBuild") && canBuildLocally(); } +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index ed07dc652..9bde4b4dc 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -30,6 +30,8 @@ public: bool canBuildLocally() const; bool willBuildLocally() const; + + bool substitutesAllowed() const; }; } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 08d1efdbe..2635e3940 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,9 +5,10 @@ #include <cerrno> #include <cstdlib> +#include <fcntl.h> #include <sys/types.h> #include <sys/stat.h> -#include <fcntl.h> +#include <sys/file.h> namespace nix { @@ -40,17 +41,14 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); @@ -58,9 +56,9 @@ bool lockFile(int fd, LockType lockType, bool wait) return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } @@ -70,14 +68,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync<StringSet> lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +81,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,75 +89,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; debug(format("locking path '%1%'") % path); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) { - if (!wait) return false; - throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); - } - lockedPaths->insert(lockPath); - } - - try { - - AutoCloseFD fd; + AutoCloseFD fd; - while (1) { + while (1) { - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - lockedPaths_.lock()->erase(lockPath); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + debug(format("lock acquired on '%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -189,8 +158,6 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on '%1%'") % i.second); @@ -208,11 +175,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index db51f950a..411da0222 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -MakeError(AlreadyLocked, Error); - class PathLocks { private: @@ -37,6 +35,4 @@ public: void setDeletion(bool deletePaths); }; -bool pathIsLockedByMe(const Path & path); - } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 5b7eb1f84..605ca9815 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -36,11 +36,10 @@ static void search(const unsigned char * s, size_t len, } if (!match) continue; string ref((const char *) s + i, refLength); - if (hashes.find(ref) != hashes.end()) { + if (hashes.erase(ref)) { debug(format("found reference to '%1%' at offset '%2%'") % ref % i); seen.insert(ref); - hashes.erase(ref); } ++i; } @@ -119,4 +118,66 @@ PathSet scanForReferences(const string & path, } +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : from(from), to(to), nextSink(nextSink) +{ + assert(from.size() == to.size()); +} + +void RewritingSink::operator () (const unsigned char * data, size_t len) +{ + std::string s(prev); + s.append((const char *) data, len); + + size_t j = 0; + while ((j = s.find(from, j)) != string::npos) { + matches.push_back(pos + j); + s.replace(j, from.size(), to); + } + + prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + + auto consumed = s.size() - prev.size(); + + pos += consumed; + + if (consumed) nextSink((unsigned char *) s.data(), consumed); +} + +void RewritingSink::flush() +{ + if (prev.empty()) return; + pos += prev.size(); + nextSink((unsigned char *) prev.data(), prev.size()); + prev.clear(); +} + +HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) + : hashSink(ht) + , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) +{ +} + +void HashModuloSink::operator () (const unsigned char * data, size_t len) +{ + rewritingSink(data, len); +} + +HashResult HashModuloSink::finish() +{ + rewritingSink.flush(); + + /* Hash the positions of the self-references. This ensures that a + NAR with self-references and a NAR with some of the + self-references already zeroed out do not produce a hash + collision. FIXME: proof. */ + for (auto & pos : rewritingSink.matches) { + auto s = fmt("|%d", pos); + hashSink((unsigned char *) s.data(), s.size()); + } + + auto h = hashSink.finish(); + return {h.first, rewritingSink.pos}; +} + } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 013809d12..c38bdd720 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,5 +7,32 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); - + +struct RewritingSink : Sink +{ + std::string from, to, prev; + Sink & nextSink; + uint64_t pos = 0; + + std::vector<uint64_t> matches; + + RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + + void operator () (const unsigned char * data, size_t len) override; + + void flush(); +}; + +struct HashModuloSink : AbstractHashSink +{ + HashSink hashSink; + RewritingSink rewritingSink; + + HashModuloSink(HashType ht, const std::string & modulus); + + void operator () (const unsigned char * data, size_t len) override; + + HashResult finish() override; +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 15faf78a5..f34369d8f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -151,7 +151,7 @@ void RemoteStore::initConnection(Connection & conn) conn.to << PROTOCOL_VERSION; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = settings.lockCPU ? lockToCurrentCPU() : -1; + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; if (cpu != -1) conn.to << 1 << cpu; else @@ -191,6 +191,14 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map<std::string, Config::SettingInfo> overrides; globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; @@ -342,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { try { std::shared_ptr<ValidPathInfo> info; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 80f18ab71..1f375dd71 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,6 +29,8 @@ public: const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(), "max-connection-age", "number of seconds to reuse a connection"}; + virtual bool sameMachine() = 0; + RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ @@ -41,7 +43,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; @@ -146,6 +148,9 @@ public: std::string getUri() override; + bool sameMachine() + { return true; } + private: ref<RemoteStore::Connection> openConnection() override; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 39205ae2c..93e117389 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -35,6 +35,9 @@ public: return uriScheme + host; } + bool sameMachine() + { return false; } + void narFromPath(const Path & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c13ff1156..1a1b24e3b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -85,18 +85,29 @@ string storePathToHash(const Path & path) void checkStoreName(const string & name) { string validChars = "+-._?="; + + auto baseError = format("The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % validChars % name; + /* Disallow names starting with a dot for possible security reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") - throw Error(format("illegal name: '%1%'") % name); + throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { - throw Error(format("invalid character '%1%' in name '%2%'") - % i % name); + throw Error(baseError % (format("the '%1%' character is invalid") % i)); } } @@ -194,15 +205,27 @@ Path Store::makeOutputPath(const string & id, } +static std::string makeType(string && type, const PathSet & references) +{ + for (auto & i : references) { + type += ":"; + type += i; + } + return type; +} + + Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const + const Hash & hash, const string & name, const PathSet & references) const { - return hash.type == htSHA256 && recursive - ? makeStorePath("source", hash, name) - : makeStorePath("output:out", hashString(htSHA256, + if (hash.type == htSHA256 && recursive) { + return makeStorePath(makeType("source", references), hash, name); + } else { + assert(references.empty()); + return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), - name); + hash.to_string(Base16) + ":"), name); + } } @@ -213,12 +236,7 @@ Path Store::makeTextPath(const string & name, const Hash & hash, /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - string type = "text"; - for (auto & i : references) { - type += ":"; - type += i; - } - return makeStorePath(type, hash, name); + return makeStorePath(makeType("text", references), hash, name); } @@ -318,13 +336,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback<ref<ValidPathInfo>> callback) + Callback<ref<ValidPathInfo>> callback) noexcept { - assertStorePath(storePath); - - auto hashPart = storePathToHash(storePath); + std::string hashPart; try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); { auto res = state.lock()->pathInfoCache.get(hashPart); @@ -354,8 +373,10 @@ void Store::queryPathInfo(const Path & storePath, } catch (...) { return callback.rethrow(); } + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -375,8 +396,8 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - callback(ref<ValidPathInfo>(info)); - } catch (...) { callback.rethrow(); } + (*callbackPtr)(ref<ValidPathInfo>(info)); + } catch (...) { callbackPtr->rethrow(); } }}); } @@ -771,8 +792,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; Hash hash(std::string(ca, recursive ? 8 : 6)); - if (references.empty() && - store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) + auto refs = references; + replaceInSet(refs, path, std::string("self")); + if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path) return true; else warn(); @@ -940,8 +962,7 @@ std::list<ref<Store>> getDefaultSubstituters() StringSet done; auto addStore = [&](const std::string & uri) { - if (done.count(uri)) return; - done.insert(uri); + if (!done.insert(uri).second) return; try { stores.push_back(openStore(uri)); } catch (Error & e) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7a1b31d0f..250e1a4bc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -303,7 +303,8 @@ public: const Hash & hash, const string & name) const; Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name) const; + const Hash & hash, const string & name, + const PathSet & references = {}) const; Path makeTextPath(const string & name, const Hash & hash, const PathSet & references) const; @@ -360,12 +361,12 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback<ref<ValidPathInfo>> callback); + Callback<ref<ValidPathInfo>> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) = 0; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0; public: @@ -628,6 +629,9 @@ public: return storePath; } + virtual void createUser(const std::string & userName, uid_t userId) + { } + protected: Stats stats; @@ -766,8 +770,7 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(), const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options like - ‘binary-caches’. */ + ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 1c14ebb18..7caee1da7 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -256,23 +256,9 @@ Hash hashString(HashType ht, const string & s) Hash hashFile(HashType ht, const Path & path) { - Ctx ctx; - Hash hash(ht); - start(ht, ctx); - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError(format("opening file '%1%'") % path); - - std::vector<unsigned char> buf(8192); - ssize_t n; - while ((n = read(fd.get(), buf.data(), buf.size()))) { - checkInterrupt(); - if (n == -1) throw SysError(format("reading file '%1%'") % path); - update(ht, ctx, buf.data(), n); - } - - finish(ht, ctx, hash.hash); - return hash; + HashSink sink(ht); + readFile(path, sink); + return sink.finish().first; } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 2dbc3b630..ffa43ecf5 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -111,7 +111,12 @@ string printHashType(HashType ht); union Ctx; -class HashSink : public BufferedSink +struct AbstractHashSink : virtual Sink +{ + virtual HashResult finish() = 0; +}; + +class HashSink : public BufferedSink, public AbstractHashSink { private: HashType ht; @@ -122,8 +127,8 @@ public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); - void write(const unsigned char * data, size_t len); - HashResult finish(); + void write(const unsigned char * data, size_t len) override; + HashResult finish() override; HashResult currentHash(); }; diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 0a6fb65f0..74e37b4c4 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -171,4 +171,9 @@ JSONObject JSONPlaceholder::object() return JSONObject(state); } +JSONPlaceholder::~JSONPlaceholder() +{ + assert(!first || std::uncaught_exception()); +} + } diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 02a39917f..83213ca66 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -168,10 +168,7 @@ public: { } - ~JSONPlaceholder() - { - assert(!first || std::uncaught_exception()); - } + ~JSONPlaceholder(); template<typename T> void write(const T & v) diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 3ccc23fd5..e41a67d1f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,4 +6,4 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 799c6e1ae..b379306f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger(); void Logger::warn(const std::string & msg) { - log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } class SimpleLogger : public Logger @@ -46,6 +46,7 @@ public: char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 678703102..5df03da74 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -6,6 +6,7 @@ namespace nix { typedef enum { lvlError = 0, + lvlWarn, lvlInfo, lvlTalkative, lvlChatty, @@ -25,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -35,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 969e4dff3..0120aeecb 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -24,7 +24,7 @@ struct Sink /* A buffered abstract sink. */ -struct BufferedSink : Sink +struct BufferedSink : virtual Sink { size_t bufSize, bufPos; std::unique_ptr<unsigned char[]> buffer; @@ -179,6 +179,36 @@ struct TeeSource : Source } }; +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; /* Convert a function into a sink. */ struct LambdaSink : Sink diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 17aee2d5c..2e416edef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -84,6 +84,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map<std::string, std::string> newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -388,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; + bytesFreed += st.st_size; if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ @@ -466,6 +475,16 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, } +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER", ""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + + static Lazy<Path> getHome2([]() { Path homeDir = getEnv("HOME"); if (homeDir.empty()) { @@ -1019,10 +1038,22 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -1047,7 +1078,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; @@ -1229,6 +1260,19 @@ string replaceStrings(const std::string & s, } +std::string rewriteStrings(const std::string & _s, const StringMap & rewrites) +{ + auto s = _s; + for (auto & i : rewrites) { + if (i.first == i.second) continue; + size_t j = 0; + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); + } + return s; +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fce3cab8d..3493e80b5 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -126,6 +126,8 @@ void deletePath(const Path & path, unsigned long long & bytesFreed); Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); +std::string getUserName(); + /* Return $HOME or the user's home directory from /etc/passwd. */ Path getHome(); @@ -270,12 +272,14 @@ struct RunOptions std::optional<uid_t> uid; std::optional<uid_t> gid; std::optional<Path> chdir; + std::optional<std::map<std::string, std::string>> environment; Path program; bool searchPath = true; Strings args; std::optional<std::string> input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) @@ -358,6 +362,20 @@ string replaceStrings(const std::string & s, const std::string & from, const std::string & to); +std::string rewriteStrings(const std::string & s, const StringMap & rewrites); + + +/* If a set contains 'from', remove it and insert 'to'. */ +template<typename T> +void replaceInSet(std::set<T> & set, const T & from, const T & to) +{ + auto i = set.find(from); + if (i == set.end()) return; + set.erase(i); + set.insert(to); +} + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); @@ -443,21 +461,34 @@ string get(const T & map, const string & key, const string & def = "") type T or an exception. (We abuse std::future<T> to pass the value or exception.) */ template<typename T> -struct Callback +class Callback { std::function<void(std::future<T>)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } - void operator()(T && t) const + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept { + auto prev = done.test_and_set(); + assert(!prev); std::promise<T> promise; promise.set_value(std::move(t)); fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) const + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept { + auto prev = done.test_and_set(); + assert(!prev); std::promise<T> promise; promise.set_exception(exc); fun(promise.get_future()); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index c6a4d4166..085650e4f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -280,14 +280,14 @@ static void _main(int argc, char * * argv) auto absolute = i; try { absolute = canonPath(absPath(i), true); - } catch (Error e) {}; + } catch (Error & e) {}; if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) - drvs.push_back(DrvInfo(*state, store, absolute)); - else - /* If we're in a #! script, interpret filenames - relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + drvs.push_back(DrvInfo(*state, store, absolute)); + else + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); } } @@ -412,7 +412,7 @@ static void _main(int argc, char * * argv) auto rcfile = (Path) tmpDir + "/rc"; writeFile(rcfile, fmt( (keepTmp ? "" : "rm -rf '%1%'; "s) + - "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 8b66cc7e3..70aa5c966 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -86,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -111,22 +113,11 @@ static void update(const StringSet & channelNames) } if (!unpacked) { - // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - // Check if the channel advertises a binary cache. - DownloadRequest request(url + "/binary-cache-url"); - try { - auto dlRes = dl->download(request); - extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; - } catch (DownloadError & e) { - } - // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } @@ -168,13 +159,7 @@ static int _main(int argc, char ** argv) nixDefExpr = home + "/.nix-defexpr"; // Figure out the name of the channels profile. - ; - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER", ""); - if (name.empty()) - throw Error("cannot figure out user name"); - profile = settings.nixStateDir + "/profiles/per-user/" + name + "/channels"; - createDirs(dirOf(profile)); + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); enum { cNone, diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8d63b8f36..cd18489b0 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -574,7 +574,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else if (setSubstituters(settings.extraSubstituters)) ; else - debug("ignoring untrusted setting '%s'", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } @@ -742,7 +742,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } -static void processConnection(bool trusted) +static void processConnection(bool trusted, + const std::string & userName, uid_t userId) { MonitorFdHup monitor(from.fd); @@ -793,6 +794,8 @@ static void processConnection(bool trusted) params["path-info-cache-size"] = "0"; auto store = openStore(settings.storeUri, params); + store->createUser(userName, userId); + tunnelLogger->stopWork(); to.flush(); @@ -1053,7 +1056,7 @@ static void daemonLoop(char * * argv) /* Handle the connection. */ from.fd = remote.get(); to.fd = remote.get(); - processConnection(trusted); + processConnection(trusted, user, peer.uid); exit(0); }, options); @@ -1133,7 +1136,7 @@ static int _main(int argc, char * * argv) } } } else { - processConnection(true); + processConnection(true, "root", 0); } } else { daemonLoop(argv); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 56ed75dae..199dc92aa 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -124,11 +124,10 @@ static void getAllExprs(EvalState & state, string attrName = i; if (hasSuffix(attrName, ".nix")) attrName = string(attrName, 0, attrName.size() - 4); - if (attrs.find(attrName) != attrs.end()) { + if (!attrs.insert(attrName).second) { printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); continue; } - attrs.insert(attrName); /* Load the expression on demand. */ Value & vFun = state.getBuiltin("import"); Value & vArg(*state.allocValue()); @@ -193,12 +192,6 @@ static void loadDerivations(EvalState & state, Path nixExprPath, } -static Path getDefNixExprPath() -{ - return getHome() + "/.nix-defexpr"; -} - - static long getPriority(EvalState & state, DrvInfo & drv) { return drv.queryMetaInt("priority", 0); @@ -307,10 +300,8 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, /* Insert only those elements in the final list that we haven't inserted before. */ for (auto & j : matches) - if (done.find(j.second) == done.end()) { - done.insert(j.second); + if (done.insert(j.second).second) elems.push_back(j.first); - } } checkSelectorUse(selectors); @@ -860,7 +851,10 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems) for (auto & i : elems) { JSONObject pkgObj = topObj.object(i.attrPath); - pkgObj.attr("name", i.queryName()); + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); JSONObject metaObj = pkgObj.object("meta"); @@ -1026,10 +1020,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (printAttrPath) columns.push_back(i.attrPath); - if (xmlOutput) - attrs["name"] = i.queryName(); - else if (printName) + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { columns.push_back(i.queryName()); + } if (compareVersions) { /* Compare this element against the versions of the @@ -1323,9 +1321,22 @@ static int _main(int argc, char * * argv) Globals globals; globals.instSource.type = srcUnknown; - globals.instSource.nixExprPath = getDefNixExprPath(); + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; globals.instSource.systemFilter = "*"; + if (!pathExists(globals.instSource.nixExprPath)) { + try { + createDirs(globals.instSource.nixExprPath); + replaceSymlink( + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } catch (Error &) { } + } + globals.dryRun = false; globals.preserveInstalled = false; globals.removeAll = false; @@ -1418,9 +1429,18 @@ static int _main(int argc, char * * argv) if (globals.profile == "") { Path profileLink = getHome() + "/.nix-profile"; - globals.profile = pathExists(profileLink) - ? absPath(readLink(profileLink), dirOf(profileLink)) - : canonPath(settings.nixStateDir + "/profiles/default"); + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + globals.profile = profileLink; + } } op(globals, opFlags, opArgs); diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index abdfa5e58..d448654fe 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -71,9 +71,7 @@ void printClosure(const Path & nePath, const StoreExpr & fs) Path path = *(workList.begin()); workList.erase(path); - if (doneSet.find(path) == doneSet.end()) { - doneSet.insert(path); - + if (doneSet.insert(path).second) { ClosureElems::const_iterator elem = fs.closure.elems.find(path); if (elem == fs.closure.elems.end()) throw Error(format("bad closure, missing path '%1%'") % path); @@ -104,8 +102,7 @@ void printDotGraph(ref<Store> store, const PathSet & roots) Path path = *(workList.begin()); workList.erase(path); - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); + if (!doneSet.insert(path).second) continue; cout << makeNode(path, symbolicName(path), "#ff0000"); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f324056bb..19384499d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -242,11 +242,10 @@ const string treeNull = " "; static void printTree(const Path & path, const string & firstPad, const string & tailPad, PathSet & done) { - if (done.find(path) != done.end()) { + if (!done.insert(path).second) { cout << format("%1%%2% [...]\n") % firstPad % path; return; } - done.insert(path); cout << format("%1%%2%\n") % firstPad % path; @@ -950,8 +949,16 @@ static void opServe(Strings opFlags, Strings opArgs) info.sigs = readStrings<StringSet>(in); in >> info.ca; - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, in, NoRepair, NoCheckSigs); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); out << 1; // indicate success diff --git a/src/nix/command.cc b/src/nix/command.cc index 3d7d582d6..532f331a7 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -129,7 +129,7 @@ void StorePathsCommand::run(ref<Store> store) } else { - for (auto & p : toStorePaths(store, NoBuild, installables)) + for (auto & p : toStorePaths(store, realiseMode, installables)) storePaths.push_back(p); if (recursive) { diff --git a/src/nix/command.hh b/src/nix/command.hh index 97a6fee7f..45ad1cd2a 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -139,6 +139,10 @@ private: bool recursive = false; bool all = false; +protected: + + RealiseMode realiseMode = NoBuild; + public: StorePathsCommand(bool recursive = false); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 96bd453d8..12a9f9cd3 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -36,7 +36,7 @@ struct CmdCopy : StorePathsCommand .set(&checkSigs, NoCheckSigs); mkFlag() - .longName("substitute") + .longName("substitute-on-destination") .shortName('s') .description("whether to try substitutes on the destination store (only supported by SSH)") .set(&substitute, Substitute); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d..a6169f118 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -55,7 +55,7 @@ struct CmdEdit : InstallableCommand int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw Error("cannot parse line number '%s'", pos); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index af4105e28..d7451376c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -2,6 +2,8 @@ #include "hash.hh" #include "legacy.hh" #include "shared.hh" +#include "references.hh" +#include "archive.hh" using namespace nix; @@ -13,6 +15,7 @@ struct CmdHash : Command bool truncate = false; HashType ht = htSHA256; std::vector<std::string> paths; + std::optional<std::string> modulus; CmdHash(Mode mode) : mode(mode) { @@ -23,6 +26,13 @@ struct CmdHash : Command mkFlag() .longName("type") .mkHashTypeFlag(&ht); + #if 0 + mkFlag() + .longName("modulo") + .description("compute hash modulo specified string") + .labels({"modulus"}) + .dest(&modulus); + #endif expectArgs("paths", &paths); } @@ -41,7 +51,19 @@ struct CmdHash : Command void run() override { for (auto path : paths) { - Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + + std::unique_ptr<AbstractHashSink> hashSink; + if (modulus) + hashSink = std::make_unique<HashModuloSink>(ht, *modulus); + else + hashSink = std::make_unique<HashSink>(ht); + + if (mode == mFile) + readFile(path, *hashSink); + else + dumpPath(path, *hashSink); + + Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % h.to_string(base, base == SRI); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0c1ad3ab3..0e8bba39d 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -39,31 +39,32 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state) auto searchPath = state.getSearchPath(); - state.mkAttrs(*vSourceExpr, searchPath.size() + 1); + state.mkAttrs(*vSourceExpr, 1024); mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); std::unordered_set<std::string> seen; - for (auto & i : searchPath) { - if (i.first == "") continue; - if (seen.count(i.first)) continue; - seen.insert(i.first); -#if 0 - auto res = state.resolveSearchPathElem(i); - if (!res.first) continue; - if (!pathExists(res.second)) continue; - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), - state.getBuiltin("import"), - mkString(*state.allocValue(), res.second)); -#endif + auto addEntry = [&](const std::string & name) { + if (name == "") return; + if (!seen.insert(name).second) return; Value * v1 = state.allocValue(); mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), i.first)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(i.first)), + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), state.getBuiltin("import"), *v2); - } + }; + + for (auto & i : searchPath) + /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto & j : readDirectory(i.second)) + if (j.name != "manifest.nix" + && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) + addEntry(j.name); + } else + addEntry(i.first); vSourceExpr->attrs->sort(); } diff --git a/src/nix/local.mk b/src/nix/local.mk index ca4604d56..c09efd1fc 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -17,7 +17,7 @@ nix_SOURCES := \ nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ diff --git a/src/nix/main.cc b/src/nix/main.cc index 4f87ad72b..22f5e8c4c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,19 +8,52 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "download.hh" #include "finally.hh" +#include <sys/types.h> +#include <sys/socket.h> +#include <ifaddrs.h> +#include <netdb.h> + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) + return true; + } + } + + return false; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; + bool useNet = true; NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { @@ -45,6 +78,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs mkFlag() .longName("print-build-logs") + .shortName('L') .description("print full build logs on stderr") .set(&printBuildLogs, true); @@ -52,6 +86,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); } void printFlags(std::ostream & out) override @@ -92,13 +131,15 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlError; + verbosity = lvlWarn; settings.verboseBuild = false; NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); + settings.requireExperimentalFeature("nix-command"); + initPlugins(); if (!args.command) args.showHelpAndExit(); @@ -107,6 +148,23 @@ void mainWrapped(int argc, char * * argv) startProgressBar(args.printBuildLogs); + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits<unsigned int>::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } + args.command->prepare(); args.command->run(); } diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc new file mode 100644 index 000000000..16344ee14 --- /dev/null +++ b/src/nix/make-content-addressable.cc @@ -0,0 +1,95 @@ +#include "command.hh" +#include "store-api.hh" +#include "references.hh" + +using namespace nix; + +struct CmdMakeContentAddressable : StorePathsCommand +{ + CmdMakeContentAddressable() + { + realiseMode = Build; + } + + std::string name() override + { + return "make-content-addressable"; + } + + std::string description() override + { + return "rewrite a path or closure to content-addressable form"; + } + + Examples examples() override + { + return { + Example{ + "To create a content-addressable representation of GNU Hello (but not its dependencies):", + "nix make-content-addressable nixpkgs.hello" + }, + Example{ + "To compute a content-addressable representation of the current NixOS system closure:", + "nix make-content-addressable -r /run/current-system" + }, + }; + } + void run(ref<Store> store, Paths storePaths) override + { + auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end())); + + paths.reverse(); + + std::map<Path, Path> remappings; + + for (auto & path : paths) { + auto oldInfo = store->queryPathInfo(path); + auto oldHashPart = storePathToHash(path); + auto name = storePathToName(path); + + StringSink sink; + store->narFromPath(path, sink); + + StringMap rewrites; + + ValidPathInfo info; + for (auto & ref : oldInfo->references) { + if (ref == path) + info.references.insert("self"); + else { + auto replacement = get(remappings, ref, ref); + // FIXME: warn about unremapped paths? + info.references.insert(replacement); + if (replacement != ref) + rewrites[storePathToHash(ref)] = storePathToHash(replacement); + } + } + + *sink.s = rewriteStrings(*sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + + info.narHash = hashModuloSink.finish().first; + info.narSize = sink.s->size(); + replaceInSet(info.references, path, std::string("self")); + info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references); + replaceInSet(info.references, std::string("self"), info.path); + info.ca = makeFixedOutputCA(true, info.narHash); + + printError("rewrote '%s' to '%s'", path, info.path); + + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + + store->addToStore(info, *source); + + remappings[path] = info.path; + } + } +}; + +static RegisterCommand r1(make_ref<CmdMakeContentAddressable>()); diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 304f918cc..5c05d6b22 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -62,6 +62,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool haveUpdate = true; }; Sync<State> state_; @@ -83,7 +84,8 @@ public: updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { - state.wait(updateCV); + if (!state->haveUpdate) + state.wait(updateCV); draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } @@ -118,7 +120,7 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { if (state.active) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { auto s2 = s + ANSI_NORMAL "\n"; @@ -168,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -178,7 +188,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(); + update(*state); } /* Check whether an activity has an ancestore with the specified @@ -213,7 +223,7 @@ public: state->its.erase(i); } - update(); + update(*state); } void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override @@ -223,42 +233,46 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(); + update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; if (printBuildLogs) { - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { state->activities.erase(i->second); info.lastLine = lastLine; state->activities.emplace_back(info); i->second = std::prev(state->activities.end()); - update(); + update(*state); } } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(); + update(*state); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(); + update(*state); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(); + update(*state); } else if (type == resProgress) { @@ -269,7 +283,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(); + update(*state); } else if (type == resSetExpected) { @@ -281,17 +295,19 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(); + update(*state); } } - void update() + void update(State & state) { + state.haveUpdate = true; updateCV.notify_one(); } void draw(State & state) { + state.haveUpdate = false; if (!state.active) return; std::string line; @@ -352,11 +368,18 @@ public: if (running || done || expected || failed) { if (running) - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - running / unit, done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); else if (expected != done) - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); else s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); s = fmt(itemFmt, s); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index d8f812149..f857b2e89 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -9,7 +9,14 @@ #include <readline/history.h> #include <readline/readline.h> #else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { #include <editline.h> +} #endif #include "shared.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index 35b763345..90b76d666 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -199,7 +199,10 @@ void chrootHelper(int argc, char * * argv) uid_t gid = getgid(); if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); /* Bind-mount realStoreDir on /nix/store. If the latter mount point doesn't already exists, we have to create a chroot diff --git a/src/nix/search.cc b/src/nix/search.cc index e086de226..eb75493e4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -80,10 +80,6 @@ struct CmdSearch : SourceExprCommand, MixJSON Example{ "To search for git and frontend or gui:", "nix search git 'frontend|gui'" - }, - Example{ - "To display the description of the found packages:", - "nix search git --verbose" } }; } @@ -265,6 +261,7 @@ struct CmdSearch : SourceExprCommand, MixJSON https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ if (!jsonCacheFile) throw Error("error writing to %s", tmpFile); + throw; } if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 7ef571561..4b0f80c62 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "sync.hh" #include "thread-pool.hh" +#include "references.hh" #include <atomic> @@ -13,7 +14,7 @@ struct CmdVerify : StorePathsCommand bool noContents = false; bool noTrust = false; Strings substituterUris; - size_t sigsNeeded; + size_t sigsNeeded = 0; CmdVerify() { @@ -88,10 +89,15 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - HashSink sink(info->narHash.type); - store->narFromPath(info->path, sink); + std::unique_ptr<AbstractHashSink> hashSink; + if (info->ca == "") + hashSink = std::make_unique<HashSink>(info->narHash.type); + else + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path)); - auto hash = sink.finish(); + store->narFromPath(info->path, *hashSink); + + auto hash = hashSink->finish(); if (hash.first != info->narHash) { corrupted++; @@ -113,13 +119,12 @@ struct CmdVerify : StorePathsCommand else { StringSet sigsSeen; - size_t actualSigsNeeded = sigsNeeded ? sigsNeeded : 1; + size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { for (auto sig : sigs) { - if (sigsSeen.count(sig)) continue; - sigsSeen.insert(sig); + if (!sigsSeen.insert(sig).second) continue; if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) validSigs++; } diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index e02cfc03e..45e8b6796 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -117,9 +117,7 @@ Path resolveSymlink(const Path & path) std::set<string> resolveTree(const Path & path, PathSet & deps) { std::set<string> results; - if (deps.count(path)) - return {}; - deps.insert(path); + if (!deps.insert(path).second) return {}; for (auto & lib : runResolver(path)) { results.insert(lib); for (auto & p : resolveTree(lib, deps)) { diff --git a/tests/build-dry.sh b/tests/build-dry.sh index 610e6070c..e72533e70 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -8,13 +8,13 @@ clearStore clearCache # Ensure this builds successfully first -nix build -f dependencies.nix +nix build --no-link -f dependencies.nix clearStore clearCache # Try --dry-run using old command first -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" # Now new command: nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" @@ -27,7 +27,7 @@ clearCache # Try --dry-run using new command first nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" # Now old command: -nix-build dependencies.nix --dry-run 2>&1 | grep "will be built" +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" fi ################################################### diff --git a/tests/check.nix b/tests/check.nix index 08aac2fb0..56c82e565 100644 --- a/tests/check.nix +++ b/tests/check.nix @@ -10,6 +10,11 @@ with import ./config.nix; ''; }; + hashmismatch = import <nix/fetchurl.nix> { + url = "file://" + toString ./dummy; + sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; + }; + fetchurl = import <nix/fetchurl.nix> { url = "file://" + toString ./lang/eval-okay-xml.exp.xml; sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v"; diff --git a/tests/check.sh b/tests/check.sh index b05e40ffb..bc23a6634 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -6,14 +6,16 @@ nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check nix-build check.nix -A nondeterministic --no-out-link -(! nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] clearStore nix-build dependencies.nix --no-out-link --repeat 3 -(! nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log) +nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? +[ "$status" = "1" ] grep 'differs from previous round' $TEST_ROOT/log path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') @@ -23,10 +25,23 @@ echo foo > $path chmod -w $path nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' - # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. [[ $(cat $path) = foo ]] nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' - [[ $(cat $path) != foo ]] + +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +echo -n > ./dummy +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +echo 'Hello World' > ./dummy + +nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +# Multiple failures with --keep-going +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +[ "$status" = "110" ] diff --git a/tests/common.sh.in b/tests/common.sh.in index 6a523ca9d..15d7b1ef9 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -16,6 +16,7 @@ if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 export NIX_REMOTE=$NIX_REMOTE_ unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237add..eca4b2964 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -17,6 +17,7 @@ let { builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/fetchurl.sh b/tests/fetchurl.sh index ec3399b08..7319ced2b 100644 --- a/tests/fetchurl.sh +++ b/tests/fetchurl.sh @@ -42,6 +42,10 @@ ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") +# Test hashed mirrors with an SRI hash. +nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ + --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + # Test unpacking a NAR. rm -rf $TEST_ROOT/archive mkdir -p $TEST_ROOT/archive diff --git a/tests/function-trace.sh b/tests/function-trace.sh new file mode 100755 index 000000000..182a4d5c2 --- /dev/null +++ b/tests/function-trace.sh @@ -0,0 +1,85 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh new file mode 100644 index 000000000..de1e2cfe4 --- /dev/null +++ b/tests/gc-auto.sh @@ -0,0 +1,70 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) +garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) +garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) + +ls -l $garbage3 +POSIXLY_CORRECT=1 du $garbage3 + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat <<EOF +with import ./config.nix; mkDerivation { + name = "gc-A"; + buildCommand = '' + set -x + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]] + mkdir \$out + echo foo > \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp1 + mv ${fake_free}.tmp1 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +expr2=$(cat <<EOF +with import ./config.nix; mkDerivation { + name = "gc-B"; + buildCommand = '' + set -x + mkdir \$out + echo foo > \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp2 + mv ${fake_free}.tmp2 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + ''; +} +EOF +) + +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/import-derivation.nix b/tests/import-derivation.nix index 91adcd288..44fa9a45d 100644 --- a/tests/import-derivation.nix +++ b/tests/import-derivation.nix @@ -10,7 +10,10 @@ let ''; }; - value = import bar; + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; in diff --git a/tests/init.sh b/tests/init.sh index 19a12c1e2..6a119aad0 100644 --- a/tests/init.sh +++ b/tests/init.sh @@ -17,6 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF build-users-group = keep-derivations = false sandbox = false +experimental-features = nix-command include nix.conf.extra EOF diff --git a/tests/install-darwin.sh b/tests/install-darwin.sh index c99ce84ac..9933eba94 100755 --- a/tests/install-darwin.sh +++ b/tests/install-darwin.sh @@ -34,7 +34,7 @@ cleanup() { sudo rm -rf /etc/nix \ /nix \ /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \ - "$USER/.nix-profile" "$USER/.nix-defexpr" "$USER/.nix-channels" + "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" } verify() { diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/lang/eval-okay-fromTOML.exp index 5b9d47122..d0dd3af2c 100644 --- a/tests/lang/eval-okay-fromTOML.exp +++ b/tests/lang/eval-okay-fromTOML.exp @@ -1 +1 @@ -[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "'key2'" = "value"; "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; g = { h = { i = { }; }; }; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { "'l'" = { }; }; }; key = "value"; name = "Orange"; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } ] +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ] diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/lang/eval-okay-fromTOML.nix index 8e7cbd1c6..963932689 100644 --- a/tests/lang/eval-okay-fromTOML.nix +++ b/tests/lang/eval-okay-fromTOML.nix @@ -46,15 +46,15 @@ "character encoding" = "value" "ʎǝʞ" = "value" 'key2' = "value" - #'quoted "value"' = "value" + 'quoted "value"' = "value" name = "Orange" - # FIXME: cpptoml doesn't handle dotted keys properly yet. - #physical.color = "orange" - #physical.shape = "round" - #site."google.com" = true + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + # This is legal according to the spec, but cpptoml doesn't handle it. #a.b.c = 1 #a.d = 2 @@ -68,16 +68,14 @@ int6 = 5_349_221 int7 = 1_2_3_4_5 - # FIXME: cpptoml doesn't support these yet: + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef - #hex1 = 0xDEADBEEF - #hex2 = 0xdeadbeef - #hex3 = 0xdead_beef + oct1 = 0o01234567 + oct2 = 0o755 - #oct1 = 0o01234567 - #oct2 = 0o755 - - #bin1 = 0b11010110 + bin1 = 0b11010110 flt1 = +1.0 flt2 = 3.1415 @@ -126,8 +124,8 @@ key1 = "another string" key2 = 456 - #[dog."tater.man"] - #type.name = "pug" + [dog."tater.man"] + type.name = "pug" [a.b.c] [ d.e.f ] @@ -137,7 +135,7 @@ name = { first = "Tom", last = "Preston-Werner" } point = { x = 1, y = 2 } - #animal = { type.name = "pug" } + animal = { type.name = "pug" } [[products]] name = "Hammer" @@ -149,6 +147,25 @@ name = "Nail" sku = 284758393 color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" '') (builtins.fromTOML '' @@ -181,4 +198,11 @@ "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" '') + + (builtins.fromTOML '' + a = [[{ b = true }]] + c = [ [ { d = true } ] ] + e = [[123]] + '') + ] diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/lang/parse-fail-mixed-nested-attrs1.nix new file mode 100644 index 000000000..11e40e66f --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.nix @@ -0,0 +1,4 @@ +{ + x.z = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/lang/parse-fail-mixed-nested-attrs2.nix new file mode 100644 index 000000000..17da82e5f --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.nix @@ -0,0 +1,4 @@ +{ + x.y.y = 3; + x = { y.y= 3; z = 3; }; +} diff --git a/tests/lang/parse-fail-dup-attrs-6.nix b/tests/lang/parse-okay-dup-attrs-6.nix index ae6d7a769..ae6d7a769 100644 --- a/tests/lang/parse-fail-dup-attrs-6.nix +++ b/tests/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/lang/parse-okay-mixed-nested-attrs-1.nix new file mode 100644 index 000000000..fd1001c8c --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/lang/parse-okay-mixed-nested-attrs-2.nix new file mode 100644 index 000000000..ad066b680 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.nix @@ -0,0 +1,4 @@ +{ + x.q = 3; + x = { y = 3; z = 3; }; +} diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/lang/parse-okay-mixed-nested-attrs-3.nix new file mode 100644 index 000000000..45a33e480 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.nix @@ -0,0 +1,7 @@ +{ + services.ssh.enable = true; + services.ssh = { port = 123; }; + services = { + httpd.enable = true; + }; +} diff --git a/tests/local.mk b/tests/local.mk index 1ff68348b..187f96ea2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,9 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - gc.sh gc-concurrent.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ @@ -26,7 +28,9 @@ nix_tests = \ check.sh \ plugins.sh \ search.sh \ - nix-copy-ssh.sh + nix-copy-ssh.sh \ + post-hook.sh \ + function-trace.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 55f1695c4..93f837bef 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -36,7 +36,7 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +[ -e $TEST_HOME/.nix-profile/foobar ] clearProfiles rm -f $TEST_HOME/.nix-channels @@ -55,5 +55,5 @@ grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml # Do an install. nix-env -i dependencies -[ -e $TEST_ROOT/var/nix/profiles/default/foobar ] +[ -e $TEST_HOME/.nix-profile/foobar ] diff --git a/tests/nix-copy-ssh.sh b/tests/nix-copy-ssh.sh index 6aba667a4..eb801548d 100644 --- a/tests/nix-copy-ssh.sh +++ b/tests/nix-copy-ssh.sh @@ -7,7 +7,7 @@ remoteRoot=$TEST_ROOT/store2 chmod -R u+w "$remoteRoot" || true rm -rf "$remoteRoot" -outPath=$(nix-build dependencies.nix) +outPath=$(nix-build --no-out-link dependencies.nix) nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index b244815e2..e2e0d1090 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -7,8 +7,3 @@ rm -rf $TEST_HOME $TEST_ROOT/profile-var mkdir -p $TEST_HOME USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency - -[ -L $TEST_HOME/.nix-profile ] -[ -e $TEST_HOME/.nix-channels ] -[ -e $TEST_ROOT/profile-var/nix/gcroots/per-user/$user ] -[ -e $TEST_ROOT/profile-var/nix/profiles/per-user/$user ] diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index 6024ea399..ee502dddb 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -27,13 +27,13 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell.drv -[[ $(nix-shell --pure $PWD/shell.drv --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --indirect --add-root shell -[[ $(nix-shell --pure shell --run \ +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p diff --git a/tests/placeholders.sh b/tests/placeholders.sh index 071cfe2dc..cd1bb7bc2 100644 --- a/tests/placeholders.sh +++ b/tests/placeholders.sh @@ -18,5 +18,3 @@ nix-build --no-out-link -E ' "; } ' - -echo XYZZY diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 000000000..a02657215 --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh new file mode 100755 index 000000000..6aadb916b --- /dev/null +++ b/tests/push-to-store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs diff --git a/tests/remote-store.sh b/tests/remote-store.sh index f2f2806d0..77437658e 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -13,3 +13,7 @@ cmp $TEST_ROOT/d1 $TEST_ROOT/d2 nix-store --gc --max-freed 1K killDaemon + +user=$(whoami) +[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] +[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff --git a/tests/timeout.sh b/tests/timeout.sh index 39ecf0a1a..eea9b5731 100644 --- a/tests/timeout.sh +++ b/tests/timeout.sh @@ -2,10 +2,14 @@ source common.sh -failed=0 -messages="`nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1 || failed=1`" -if [ $failed -ne 0 ]; then - echo "error: 'nix-store' succeeded; should have timed out" + +set +e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) +status=$? +set -e + +if [ $status -ne 101 ]; then + echo "error: 'nix-store' exited with '$status'; should have exited 101" exit 1 fi diff --git a/tests/user-envs.sh b/tests/user-envs.sh index ba6392311..aebf6a2a2 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -20,7 +20,7 @@ drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1 # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly -rm -f $HOME/.nix-defexpr +rm -rf $HOME/.nix-defexpr ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr nix-env -qa '*' --description | grep -q silly |