diff options
30 files changed, 589 insertions, 61 deletions
diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py new file mode 100755 index 000000000..ee77e1589 --- /dev/null +++ b/contrib/stack-collapse.py @@ -0,0 +1,39 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `-vvvv` and +# `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls -vvvv '<nixpkgs>' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index c7b540892..320e15339 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -873,6 +873,14 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term> + <listitem> + <para>The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration.</para> + </listitem> + </varlistentry> + <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by @@ -973,6 +981,34 @@ requiredSystemFeatures = [ "kvm" ]; </varlistentry> + <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term> + + <listitem> + + <para>Default: <literal>false</literal>.</para> + + <para>If set to <literal>true</literal>, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit.</para> + + <informalexample><screen> +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 +</screen></informalexample> + + <para>The <literal>undefined position</literal> means the function + call is a builtin.</para> + + <para>Use the <literal>contrib/stack-collapse.py</literal> script + distributed with the Nix source code to convert the trace logs + in to a format suitable for <command>flamegraph.pl</command>.</para> + + </listitem> + + </varlistentry> + <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term> <listitem><para>A whitespace-separated list of public keys. When diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 56c466268..c8c01f9a4 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -221,31 +221,53 @@ also <xref linkend="sec-common-options" />.</phrase></para> <varlistentry><term><filename>~/.nix-defexpr</filename></term> - <listitem><para>A directory that contains the default Nix + <listitem><para>The source for the default Nix expressions used by the <option>--install</option>, <option>--upgrade</option>, and <option>--query - --available</option> operations to obtain derivations. The + --available</option> operations to obtain derivations. The <option>--file</option> option may be used to override this default.</para> - <para>The Nix expressions in this directory are combined into a - single set, with each file as an attribute that has the name of - the file. Thus, if <filename>~/.nix-defexpr</filename> contains - two files, <filename>foo</filename> and <filename>bar</filename>, + <para>If <filename>~/.nix-defexpr</filename> is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory + containing a <filename>default.nix</filename> file, that file + is loaded as in the above paragraph.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory without + a <filename>default.nix</filename> file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + </para> + + <para>For example, if <filename>~/.nix-defexpr</filename> contains + two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>, then the default Nix expression will essentially be <programlisting> { - foo = import ~/.nix-defexpr/foo; - bar = import ~/.nix-defexpr/bar; + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; }</programlisting> </para> + <para>The file <filename>manifest.nix</filename> is always ignored. + Subdirectories without a <filename>default.nix</filename> file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression.</para> + <para>The command <command>nix-channel</command> places symlinks to the downloaded Nix expressions from each subscribed channel in this directory.</para> - </listitem> </varlistentry> @@ -1348,10 +1370,13 @@ profile. The generations can be a list of generation numbers, the special value <literal>old</literal> to delete all non-current generations, a value such as <literal>30d</literal> to delete all generations older than the specified number of days (except for the -generation that was active at that point in time), or a value such as. -<literal>+5</literal> to only keep the specified items older than the -current generation. Periodically deleting old generations is important -to make garbage collection effective.</para> +generation that was active at that point in time), or a value such as +<literal>+5</literal> to keep the last <literal>5</literal> generations +ignoring any newer than current, e.g., if <literal>30</literal> is the current +generation <literal>+5</literal> will delete generation <literal>25</literal> +and all older generations. +Periodically deleting old generations is important to make garbage collection +effective.</para> </refsection> diff --git a/doc/manual/expressions/language-operators.xml b/doc/manual/expressions/language-operators.xml index f1f750934..4f11bf529 100644 --- a/doc/manual/expressions/language-operators.xml +++ b/doc/manual/expressions/language-operators.xml @@ -15,13 +15,16 @@ weakest binding).</para> <tgroup cols='3'> <thead> <row> + <entry>Name</entry> <entry>Syntax</entry> <entry>Associativity</entry> <entry>Description</entry> + <entry>Precedence</entry> </row> </thead> <tbody> <row> + <entry>Select</entry> <entry><replaceable>e</replaceable> <literal>.</literal> <replaceable>attrpath</replaceable> [ <literal>or</literal> <replaceable>def</replaceable> ] @@ -33,19 +36,25 @@ weakest binding).</para> dot-separated list of attribute names.) If the attribute doesn’t exist, return <replaceable>def</replaceable> if provided, otherwise abort evaluation.</entry> + <entry>1</entry> </row> <row> + <entry>Application</entry> <entry><replaceable>e1</replaceable> <replaceable>e2</replaceable></entry> <entry>left</entry> <entry>Call function <replaceable>e1</replaceable> with argument <replaceable>e2</replaceable>.</entry> + <entry>2</entry> </row> <row> + <entry>Arithmetic Negation</entry> <entry><literal>-</literal> <replaceable>e</replaceable></entry> <entry>none</entry> <entry>Arithmetic negation.</entry> + <entry>3</entry> </row> <row> + <entry>Has Attribute</entry> <entry><replaceable>e</replaceable> <literal>?</literal> <replaceable>attrpath</replaceable></entry> <entry>none</entry> @@ -53,34 +62,69 @@ weakest binding).</para> the attribute denoted by <replaceable>attrpath</replaceable>; return <literal>true</literal> or <literal>false</literal>.</entry> + <entry>4</entry> </row> <row> + <entry>List Concatenation</entry> <entry><replaceable>e1</replaceable> <literal>++</literal> <replaceable>e2</replaceable></entry> <entry>right</entry> <entry>List concatenation.</entry> + <entry>5</entry> </row> <row> + <entry>Multiplication</entry> <entry> <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>, + </entry> + <entry>left</entry> + <entry>Arithmetic multiplication.</entry> + <entry>6</entry> + </row> + <row> + <entry>Division</entry> + <entry> <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable> </entry> <entry>left</entry> - <entry>Arithmetic multiplication and division.</entry> + <entry>Arithmetic division.</entry> + <entry>6</entry> + </row> + <row> + <entry>Addition</entry> + <entry> + <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable> + </entry> + <entry>left</entry> + <entry>Arithmetic addition.</entry> + <entry>7</entry> </row> <row> + <entry>Subtraction</entry> <entry> - <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable>, <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable> </entry> <entry>left</entry> - <entry>Arithmetic addition and subtraction. String or path concatenation (only by <literal>+</literal>).</entry> + <entry>Arithmetic subtraction.</entry> + <entry>7</entry> </row> <row> + <entry>String Concatenation</entry> + <entry> + <replaceable>string1</replaceable> <literal>+</literal> <replaceable>string2</replaceable> + </entry> + <entry>left</entry> + <entry>String concatenation.</entry> + <entry>7</entry> + </row> + <row> + <entry>Not</entry> <entry><literal>!</literal> <replaceable>e</replaceable></entry> <entry>none</entry> <entry>Boolean negation.</entry> + <entry>8</entry> </row> <row> + <entry>Update</entry> <entry><replaceable>e1</replaceable> <literal>//</literal> <replaceable>e2</replaceable></entry> <entry>right</entry> @@ -89,47 +133,90 @@ weakest binding).</para> <replaceable>e2</replaceable> (with the latter taking precedence over the former in case of equally named attributes).</entry> + <entry>9</entry> </row> <row> + <entry>Less Than</entry> <entry> <replaceable>e1</replaceable> <literal><</literal> <replaceable>e2</replaceable>, - <replaceable>e1</replaceable> <literal>></literal> <replaceable>e2</replaceable>, - <replaceable>e1</replaceable> <literal><=</literal> <replaceable>e2</replaceable>, + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Less Than or Equal To</entry> + <entry> + <replaceable>e1</replaceable> <literal><=</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Greater Than</entry> + <entry> + <replaceable>e1</replaceable> <literal>></literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Greater Than or Equal To</entry> + <entry> <replaceable>e1</replaceable> <literal>>=</literal> <replaceable>e2</replaceable> </entry> <entry>none</entry> <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Equality</entry> + <entry> + <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Equality.</entry> + <entry>11</entry> </row> <row> + <entry>Inequality</entry> <entry> - <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable>, <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable> </entry> <entry>none</entry> - <entry>Equality and inequality.</entry> + <entry>Inequality.</entry> + <entry>11</entry> </row> <row> + <entry>Logical AND</entry> <entry><replaceable>e1</replaceable> <literal>&&</literal> <replaceable>e2</replaceable></entry> <entry>left</entry> <entry>Logical AND.</entry> + <entry>12</entry> </row> <row> + <entry>Logical OR</entry> <entry><replaceable>e1</replaceable> <literal>||</literal> <replaceable>e2</replaceable></entry> <entry>left</entry> <entry>Logical OR.</entry> + <entry>13</entry> </row> <row> + <entry>Logical Implication</entry> <entry><replaceable>e1</replaceable> <literal>-></literal> <replaceable>e2</replaceable></entry> <entry>none</entry> <entry>Logical implication (equivalent to <literal>!<replaceable>e1</replaceable> || <replaceable>e2</replaceable></literal>).</entry> + <entry>14</entry> </row> </tbody> </tgroup> </table> -</section>
\ No newline at end of file +</section> diff --git a/doc/manual/installation/env-variables.xml b/doc/manual/installation/env-variables.xml index d1ee0bb2e..e2b8fc867 100644 --- a/doc/manual/installation/env-variables.xml +++ b/doc/manual/installation/env-variables.xml @@ -67,5 +67,23 @@ $ sudo launchctl kickstart -k system/org.nixos.nix-daemon </screen> </section> +<section xml:id="sec-installer-proxy-settings"> + +<title>Proxy Environment Variables</title> + +<para>The Nix installer has special handling for these proxy-related +environment variables: +<varname>http_proxy</varname>, <varname>https_proxy</varname>, +<varname>ftp_proxy</varname>, <varname>no_proxy</varname>, +<varname>HTTP_PROXY</varname>, <varname>HTTPS_PROXY</varname>, +<varname>FTP_PROXY</varname>, <varname>NO_PROXY</varname>. +</para> +<para>If any of these variables are set when running the Nix installer, +then the installer will create an override file at +<filename>/etc/systemd/system/nix-daemon.service.d/override.conf</filename> +so <command>nix-daemon</command> will use them. +</para> +</section> + </section> </chapter> diff --git a/doc/manual/release-notes/rl-2.3.xml b/doc/manual/release-notes/rl-2.3.xml index 428213b36..a1f4a4b75 100644 --- a/doc/manual/release-notes/rl-2.3.xml +++ b/doc/manual/release-notes/rl-2.3.xml @@ -4,9 +4,23 @@ version="5.0" xml:id="ssec-relnotes-2.3"> -<title>Release 2.3 (????-??-??)</title> +<title>Release 2.3 (2019-08-??)</title> -<para>This release contains the following changes:</para> +<para>This is primarily a bug fix release. However, it makes some +incompatible changes:</para> + +<itemizedlist> + + <listitem> + <para>Nix now uses BSD file locks instead of POSIX file + locks. Since previous releases used POSIX file locks, you should + not use Nix 2.2 and previous releases at the same time on a Nix + store.</para> + </listitem> + +</itemizedlist> + +<para>It also has the following changes:</para> <itemizedlist> @@ -18,5 +32,62 @@ already begin with <literal>refs/</literal>. </para> </listitem> + + <listitem> + <para>The installer now enables sandboxing by default on + Linux. The <literal>max-jobs</literal> setting now defaults to + 1.</para> + </listitem> + + <listitem> + <para>New builtin functions: + <literal>builtins.isPath</literal>, + <literal>builtins.hashFile</literal>. + </para> + </listitem> + + <listitem> + <para><command>nix</command>: Add + <option>--print-build-logs</option> (<option>-L</option>) flag to + print build log output to stderr rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + </para> + </listitem> + + <listitem> + <para>Builds are now executed in a pseudo-terminal, and the + <envar>TERM</envar> evnironment variable is set to + <literal>xterm-256color</literal>. This allows many programs + (e.g. <command>gcc</command>, <command>clang</command>, + <command>cmake</command>) to print colorized log output.</para> + </listitem> + + <listitem> + <para>Add <option>--no-net</option> convenience flag. This flag + disables substituters; sets the <literal>tarball-ttl</literal> + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces.</para> + </listitem> + + <listitem> + <para>Add a <literal>post-build-hook</literal> setting to run a + program after a build has succeeded.</para> + </listitem> + + <listitem> + <para>Add a <literal>trace-function-calls</literal> setting to log + the duration of Nix function calls to stderr.</para> + </listitem> + + <listitem> + <para>On Linux, sandboxing is now disabled by default on systems + that don’t have the necessary kernel support.</para> + </listitem> + </itemizedlist> + </section> diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 5fc04a3f5..25655204d 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -7,3 +7,6 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/release.nix b/release.nix index 28bf7897f..d38d3819a 100644 --- a/release.nix +++ b/release.nix @@ -73,7 +73,12 @@ let # https://github.com/NixOS/nixpkgs/issues/45462 '' mkdir -p $out/lib - cp ${boost}/lib/libboost_context* $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} ''; configureFlags = configureFlags ++ @@ -166,10 +171,10 @@ let chmod +x $TMPDIR/install-systemd-multi-user.sh chmod +x $TMPDIR/install-multi-user dir=nix-${version}-${system} - fn=$out/$dir.tar.bz2 + fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfj $fn \ + tar cvfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --absolute-names \ --hard-dereference \ @@ -302,7 +307,7 @@ let substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.bz2) ") + (system: "--replace '@binaryTarball_${system}@' $(nix hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ") [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] } \ --replace '@nixVersion@' ${build.x86_64-linux.src.version} diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 9b757e7da..a41309e93 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -330,7 +330,7 @@ EOF fi done - if [ -d /nix ]; then + if [ -d /nix/store ] || [ -d /nix/var ]; then failure <<EOF There are some relics of a previous installation of Nix at /nix, and this scripts assumes Nix is _not_ yet installed. Please delete the old @@ -758,9 +758,13 @@ main() { if [ "$(uname -s)" = "Darwin" ]; then # shellcheck source=./install-darwin-multi-user.sh . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" - elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then - # shellcheck source=./install-systemd-multi-user.sh - . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi else failure "Sorry, I don't know what to do on $(uname)" fi diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 04bc539a1..bef3ac4f9 100644..100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -9,6 +9,38 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + +# Path for the systemd override unit file to contain the proxy settings +readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf + +create_systemd_override() { + header "Configuring proxy for the nix-daemon service" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE" +[Service] +$1 +EOF +} + +# Gather all non-empty proxy environment variables into a string +create_systemd_proxy_env() { + vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + for v in $vars; do + if [ "x${!v:-}" != "x" ]; then + echo "Environment=${v}=${!v}" + fi + done +} + +handle_network_proxy() { + # Create a systemd unit override with proxy environment variables + # if any proxy environment variables are not empty. + PROXY_ENV_STRING=$(create_systemd_proxy_env) + if [ -n "${PROXY_ENV_STRING}" ]; then + create_systemd_override "${PROXY_ENV_STRING}" + fi +} + poly_validate_assumptions() { if [ "$(uname -s)" != "Linux" ]; then failure "This script is for use with Linux!" @@ -47,6 +79,8 @@ poly_configure_nix_daemon_service() { _sudo "to set up the nix-daemon socket service" \ systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC" + handle_network_proxy + _sudo "to load the systemd unit for nix-daemon" \ systemctl daemon-reload diff --git a/scripts/install.in b/scripts/install.in index 4857638c0..902758b13 100644 --- a/scripts/install.in +++ b/scripts/install.in @@ -30,12 +30,11 @@ case "$(uname -s).$(uname -m)" in *) oops "sorry, there is no binary distribution of Nix for your platform";; esac -url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.bz2" +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" -tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.bz2")" +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" require_util curl "download the binary tarball" -require_util bzcat "decompress the binary tarball" require_util tar "unpack the binary tarball" echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." @@ -57,7 +56,7 @@ fi unpack=$tmpDir/unpack mkdir -p "$unpack" -< "$tarball" bzcat | tar -xf - -C "$unpack" || oops "failed to unpack '$url'" +tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" script=$(echo "$unpack"/*/install) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 46c622ee8..faa76f1f7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -10,6 +10,7 @@ #include "flake/flake.hh" #include <algorithm> +#include <chrono> #include <cstring> #include <unistd.h> #include <sys/time.h> @@ -17,7 +18,6 @@ #include <iostream> #include <fstream> -#include <sys/time.h> #include <sys/resource.h> #if HAVE_BOEHMGC @@ -1103,9 +1103,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + std::optional<FunctionCallTrace> trace; + if (evalSettings.traceFunctionCalls) { + trace.emplace(pos); + } + forceValue(fun, pos); if (fun.type == tPrimOp || fun.type == tPrimOpApp) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 46c6ea271..75e91e6b1 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,6 +6,7 @@ #include "symbol-table.hh" #include "hash.hh" #include "config.hh" +#include "function-trace.hh" #include <map> #include <unordered_map> @@ -373,6 +374,9 @@ struct EvalSettings : Config Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; + Setting<std::string> flakeRegistry{this, "https://raw.githubusercontent.com/NixOS/flake-registry/master/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; }; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 000000000..8234b7603 --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "eval.hh" +#include <sys/time.h> + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + + FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + vomit("function-trace entered %1% at %2%", pos, ns.count()); + } + + ~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + vomit("function-trace exited %1% at %2%", pos, ns.count()); + } +}; +} diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index cd752f467..0afddfb78 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -125,6 +125,15 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index db7300c58..19cb9addc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2357,17 +2357,37 @@ void DerivationGoal::startBuilder() flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) + if (child == -1 && errno == EINVAL) { /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unpriveleged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); _exit(0); }, options); - if (helper.wait() != 0) + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + tmpDirInSandbox = tmpDir; + goto fallback; + } else if (res != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2398,6 +2418,7 @@ void DerivationGoal::startBuilder() } else #endif { + fallback: options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index aec68d627..3835727a0 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -236,8 +236,6 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } - long lowSpeedTimeout = 300; - void init() { if (!req) req = curl_easy_init(); @@ -297,7 +295,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ diff --git a/src/libstore/download.hh b/src/libstore/download.hh index c095ad053..abc4a828c 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -24,6 +24,9 @@ struct DownloadSettings : Config Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + Setting<unsigned int> tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 366dbfb0a..a166f4ee2 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -690,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state) throw SysError(format("statting '%1%'") % path); if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; continue; } @@ -701,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting '%1%'") % path); - state.results.bytesFreed += st.st_blocks * 512ULL; + state.results.bytesFreed += st.st_size; } struct stat st; @@ -920,11 +919,11 @@ void LocalStore::autoGC(bool sync) promise.set_value(); }); - printInfo("running auto-GC to free %d bytes", settings.maxFree - avail); - GCOptions options; options.maxFreed = settings.maxFree - avail; + printInfo("running auto-GC to free %d bytes", options.maxFreed); + GCResults results; collectGarbage(options, results); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e845c29b0..c0c535a12 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -209,6 +209,9 @@ public: "The paths to make available inside the build sandbox.", {"build-chroot-dirs", "build-sandbox-paths"}}; + Setting<bool> sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 15faf78a5..1c2e23f9c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -191,6 +191,13 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map<std::string, Config::SettingInfo> overrides; globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e9042443c..d007af037 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -97,6 +97,10 @@ void checkStoreName(const string & name) reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 969e4dff3..a344a5ac7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -179,6 +179,36 @@ struct TeeSource : Source } }; +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; /* Convert a function into a sink. */ struct LambdaSink : Sink diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e353290d0..98a7ea397 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -397,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; + bytesFreed += st.st_size; if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f324056bb..0cbceb02f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -950,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs) info.sigs = readStrings<StringSet>(in); in >> info.ca; - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, in, NoRepair, NoCheckSigs); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); out << 1; // indicate success diff --git a/src/nix/run.cc b/src/nix/run.cc index 9c15b6749..01ec9a6f8 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -255,7 +255,10 @@ void chrootHelper(int argc, char * * argv) uid_t gid = getgid(); if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); /* Bind-mount realStoreDir on /nix/store. If the latter mount point doesn't already exists, we have to create a chroot diff --git a/tests/function-trace.sh b/tests/function-trace.sh new file mode 100755 index 000000000..f7d93b435 --- /dev/null +++ b/tests/function-trace.sh @@ -0,0 +1,86 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + -vvvv \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh index e770e665c..06f5f0795 100644 --- a/tests/gc-auto.sh +++ b/tests/gc-auto.sh @@ -2,9 +2,12 @@ source common.sh clearStore -garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) -garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) -garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) +garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) +garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) +garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) + +ls -l $garbage3 +POSIXLY_CORRECT=1 du $garbage3 fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free @@ -19,7 +22,7 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 100 > $fake_free + echo 200 > $fake_free echo 2... sleep 2 echo 3... @@ -29,7 +32,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build --impure -o $TEST_ROOT/result-A -L "($expr)" \ +nix build --impure -v -o $TEST_ROOT/result-A -L "($expr)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 & pid=$! @@ -41,7 +44,7 @@ with import ./config.nix; mkDerivation { echo foo > \$out/bar echo 1... sleep 2 - echo 100 > $fake_free + echo 200 > $fake_free echo 2... sleep 2 echo 3... @@ -50,7 +53,7 @@ with import ./config.nix; mkDerivation { EOF ) -nix build --impure -o $TEST_ROOT/result-B -L "($expr2)" \ +nix build --impure -v -o $TEST_ROOT/result-B -L "($expr2)" \ --min-free 1000 --max-free 2000 --min-free-check-interval 1 wait "$pid" diff --git a/tests/local.mk b/tests/local.mk index f4ef981fd..97c24a78e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -30,6 +30,7 @@ nix_tests = \ search.sh \ nix-copy-ssh.sh \ post-hook.sh \ + function-trace.sh \ flakes.sh # parallel.sh diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh index d97eb095d..6aadb916b 100755 --- a/tests/push-to-store.sh +++ b/tests/push-to-store.sh @@ -1,4 +1,4 @@ #!/bin/sh echo Pushing "$@" to "$REMOTE_STORE" -echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs +printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs |