diff options
29 files changed, 754 insertions, 159 deletions
diff --git a/contrib/stack-collapse.py b/contrib/stack-collapse.py new file mode 100755 index 000000000..ee77e1589 --- /dev/null +++ b/contrib/stack-collapse.py @@ -0,0 +1,39 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `-vvvv` and +# `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls -vvvv '<nixpkgs>' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/doc/manual/advanced-topics/advanced-topics.xml b/doc/manual/advanced-topics/advanced-topics.xml index 12a826254..871b7eb1d 100644 --- a/doc/manual/advanced-topics/advanced-topics.xml +++ b/doc/manual/advanced-topics/advanced-topics.xml @@ -9,5 +9,6 @@ <xi:include href="distributed-builds.xml" /> <xi:include href="cores-vs-jobs.xml" /> <xi:include href="diff-hook.xml" /> +<xi:include href="post-build-hook.xml" /> </part> diff --git a/doc/manual/advanced-topics/post-build-hook.xml b/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000..3dc43ee79 --- /dev/null +++ b/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="chap-post-build-hook" + version="5.0" + > + +<title>Using the <xref linkend="conf-post-build-hook" /></title> +<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle> + + +<section xml:id="chap-post-build-hook-caveats"> + <title>Implementation Caveats</title> + <para>Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases.</para> + + <para>The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails.</para> + + <para>Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable.</para> + + <para>A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop.</para> +</section> + +<section> + <title>Prerequisites</title> + + <para> + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + <xref linkend="ssec-s3-substituter-authenticated-writes" />, and + that the <literal>root</literal> user's default AWS profile can + upload to the bucket. + </para> +</section> + +<section> + <title>Set up a Signing Key</title> + <para>Use <command>nix-store --generate-binary-cache-key</command> to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths.</para> + + <screen> +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +</screen> + +<para>Then, add the public key and the cache URL to your +<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" /> +and <xref linkend="conf-substituters" /> like:</para> + +<programlisting> +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +</programlisting> + +<para>we will restart the Nix daemon a later step.</para> +</section> + +<section> + <title>Implementing the build hook</title> + <para>Write the following script to + <filename>/etc/nix/upload-to-cache.sh</filename>: + </para> + + <programlisting> +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS +</programlisting> + + <note> + <title>Should <literal>$OUT_PATHS</literal> be quoted?</title> + <para> + The <literal>$OUT_PATHS</literal> variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to <command>nix sign-paths</command>. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The <command>set -f</command> + disables globbing in the shell. + </para> + </note> + <para> + Then make sure the hook program is executable by the <literal>root</literal> user: + <screen> +# chmod +x /etc/nix/upload-to-cache.sh +</screen></para> +</section> + +<section> + <title>Updating Nix Configuration</title> + + <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook, + by adding the following configuration snippet at the end:</para> + + <programlisting> +post-build-hook = /etc/nix/upload-to-cache.sh +</programlisting> + +<para>Then, restart the <command>nix-daemon</command>.</para> +</section> + +<section> + <title>Testing</title> + + <para>Build any derivation, for example:</para> + + <screen> +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +</screen> + + <para>Then delete the path from the store, and try substituting it from the binary cache:</para> + <screen> +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +</screen> + +<para>Now, copy the path back from the cache:</para> +<screen> +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example +</screen> +</section> +<section> + <title>Conclusion</title> + <para> + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + </para> + + <para> + Before deploying this to production, be sure to consider the + implementation caveats in <xref linkend="chap-post-build-hook-caveats" />. + </para> +</section> +</chapter> diff --git a/doc/manual/command-ref/conf-file.xml b/doc/manual/command-ref/conf-file.xml index 4f7393e2e..320e15339 100644 --- a/doc/manual/command-ref/conf-file.xml +++ b/doc/manual/command-ref/conf-file.xml @@ -483,8 +483,10 @@ builtins.fetchurl { <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term> - <listitem><para>This option defines after how many free bytes to stop collecting - garbage once the <literal>min-free</literal> condition gets triggered.</para></listitem> + <listitem><para>When a garbage collection is triggered by the + <literal>min-free</literal> option, it stops as soon as + <literal>max-free</literal> bytes are available. The default is + infinity (i.e. delete all garbage).</para></listitem> </varlistentry> @@ -528,9 +530,11 @@ builtins.fetchurl { <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term> <listitem> - <para>When the disk reaches <literal>min-free</literal> bytes of free disk space during a build, nix - will start to garbage-collection until <literal>max-free</literal> bytes are available on the disk. - A value of <literal>0</literal> (the default) means that this feature is disabled.</para> + <para>When free disk space in <filename>/nix/store</filename> + drops below <literal>min-free</literal> during a build, Nix + performs a garbage-collection until <literal>max-free</literal> + bytes are available or there is no more garbage. A value of + <literal>0</literal> (the default) disables this feature.</para> </listitem> </varlistentry> @@ -660,6 +664,62 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-post-build-hook"> + <term><literal>post-build-hook</literal></term> + <listitem> + <para>Optional. The path to a program to execute after each build.</para> + + <para>This option is only settable in the global + <filename>nix.conf</filename>, or on the command line by trusted + users.</para> + + <para>When using the nix-daemon, the daemon executes the hook as + <literal>root</literal>. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build.</para> + + <itemizedlist> + <listitem><para>The hook executes after an evaluation-time build.</para></listitem> + <listitem><para>The hook does not execute on substituted paths.</para></listitem> + <listitem><para>The hook's output always goes to the user's terminal.</para></listitem> + <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem> + <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem> + </itemizedlist> + + <para>The program executes with no arguments. The program's environment + contains the following environment variables:</para> + + <variablelist> + <varlistentry> + <term><envar>DRV_PATH</envar></term> + <listitem> + <para>The derivation for the built paths.</para> + <para>Example: + <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><envar>OUT_PATHS</envar></term> + <listitem> + <para>Output paths of the built derivation, separated by a space character.</para> + <para>Example: + <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>See <xref linkend="chap-post-build-hook" /> for an example + implementation.</para> + + </listitem> + </varlistentry> + <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term> <listitem><para>How many times to repeat builds to check whether @@ -813,6 +873,14 @@ password <replaceable>my-password</replaceable> </varlistentry> + <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term> + <listitem> + <para>The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration.</para> + </listitem> + </varlistentry> + <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term> <listitem><para>A list of URLs of substituters, separated by @@ -913,6 +981,34 @@ requiredSystemFeatures = [ "kvm" ]; </varlistentry> + <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term> + + <listitem> + + <para>Default: <literal>false</literal>.</para> + + <para>If set to <literal>true</literal>, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit.</para> + + <informalexample><screen> +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 +</screen></informalexample> + + <para>The <literal>undefined position</literal> means the function + call is a builtin.</para> + + <para>Use the <literal>contrib/stack-collapse.py</literal> script + distributed with the Nix source code to convert the trace logs + in to a format suitable for <command>flamegraph.pl</command>.</para> + + </listitem> + + </varlistentry> + <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term> <listitem><para>A whitespace-separated list of public keys. When diff --git a/doc/manual/command-ref/nix-env.xml b/doc/manual/command-ref/nix-env.xml index 56c466268..693f23f7f 100644 --- a/doc/manual/command-ref/nix-env.xml +++ b/doc/manual/command-ref/nix-env.xml @@ -221,31 +221,53 @@ also <xref linkend="sec-common-options" />.</phrase></para> <varlistentry><term><filename>~/.nix-defexpr</filename></term> - <listitem><para>A directory that contains the default Nix + <listitem><para>The source for the default Nix expressions used by the <option>--install</option>, <option>--upgrade</option>, and <option>--query - --available</option> operations to obtain derivations. The + --available</option> operations to obtain derivations. The <option>--file</option> option may be used to override this default.</para> - <para>The Nix expressions in this directory are combined into a - single set, with each file as an attribute that has the name of - the file. Thus, if <filename>~/.nix-defexpr</filename> contains - two files, <filename>foo</filename> and <filename>bar</filename>, + <para>If <filename>~/.nix-defexpr</filename> is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory + containing a <filename>default.nix</filename> file, that file + is loaded as in the above paragraph.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory without + a <filename>default.nix</filename> file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + </para> + + <para>For example, if <filename>~/.nix-defexpr</filename> contains + two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>, then the default Nix expression will essentially be <programlisting> { - foo = import ~/.nix-defexpr/foo; - bar = import ~/.nix-defexpr/bar; + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; }</programlisting> </para> + <para>The file <filename>manifest.nix</filename> is always ignored. + Subdirectories without a <filename>default.nix</filename> file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression.</para> + <para>The command <command>nix-channel</command> places symlinks to the downloaded Nix expressions from each subscribed channel in this directory.</para> - </listitem> </varlistentry> diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8e10d9f2..9f4b6b411 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,6 +9,7 @@ #include "json.hh" #include <algorithm> +#include <chrono> #include <cstring> #include <unistd.h> #include <sys/time.h> @@ -16,7 +17,6 @@ #include <iostream> #include <fstream> -#include <sys/time.h> #include <sys/resource.h> #if HAVE_BOEHMGC @@ -1094,9 +1094,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + std::optional<FunctionCallTrace> trace; + if (evalSettings.traceFunctionCalls) { + trace.emplace(pos); + } + forceValue(fun, pos); if (fun.type == tPrimOp || fun.type == tPrimOpApp) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a314e01e0..22472fd72 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,6 +6,7 @@ #include "symbol-table.hh" #include "hash.hh" #include "config.hh" +#include "function-trace.hh" #include <map> #include <unordered_map> @@ -349,6 +350,9 @@ struct EvalSettings : Config Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 000000000..8234b7603 --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "eval.hh" +#include <sys/time.h> + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + + FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + vomit("function-trace entered %1% at %2%", pos, ns.count()); + } + + ~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + vomit("function-trace exited %1% at %2%", pos, ns.count()); + } +}; +} diff --git a/src/libstore/build.cc b/src/libstore/build.cc index cf6428e12..4bec37e0f 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1629,6 +1629,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map<std::string, std::string> hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; @@ -3984,17 +4039,6 @@ void SubstitutionGoal::tryToRun() return; } - /* If the store path is already locked (probably by a - DerivationGoal), then put this goal to sleep. Note: we don't - acquire a lock here since that breaks addToStore(), so below we - handle an AlreadyLocked exception from addToStore(). The check - here is just an optimisation to prevent having to redo a - download due to a locked path. */ - if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { - worker.waitForAWhile(shared_from_this()); - return; - } - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -4034,12 +4078,6 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (AlreadyLocked & e) { - /* Probably a DerivationGoal is already building this store - path. Sleep for a while and try again. */ - state = &SubstitutionGoal::init; - worker.waitForAWhile(shared_from_this()); - return; } catch (std::exception & e) { printError(e.what()); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 91087eebc..c322d267d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -236,8 +236,6 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } - long lowSpeedTimeout = 300; - void init() { if (!req) req = curl_easy_init(); @@ -297,7 +295,7 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ diff --git a/src/libstore/download.hh b/src/libstore/download.hh index 3b7fff3ba..c68381846 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -24,6 +24,9 @@ struct DownloadSettings : Config Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + Setting<unsigned int> tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 83cdf13a6..366dbfb0a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); @@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } @@ -221,26 +221,22 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); //if (*fd == -1) continue; - if (path != fnTempRoots) { - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + /* Read the entire file. */ string contents = readFile(fd->get()); @@ -871,7 +867,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - auto getAvail = [this]() { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); @@ -892,7 +893,7 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; auto avail = getAvail(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0af8215d1..13effb507 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -315,6 +315,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting<std::string> postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; @@ -342,6 +345,9 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 485fdd691..63b11467e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1210,7 +1210,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); PathSet store; @@ -1221,13 +1222,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + fdGCLock = -1; + for (auto & i : validPaths2) verifyPath(i, store, done, validPaths, repair, errors); - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock = -1; - /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking hashes..."); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6b655647b..af8b84bf5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -263,7 +263,7 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 08d1efdbe..2635e3940 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,9 +5,10 @@ #include <cerrno> #include <cstdlib> +#include <fcntl.h> #include <sys/types.h> #include <sys/stat.h> -#include <fcntl.h> +#include <sys/file.h> namespace nix { @@ -40,17 +41,14 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); @@ -58,9 +56,9 @@ bool lockFile(int fd, LockType lockType, bool wait) return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } @@ -70,14 +68,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync<StringSet> lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +81,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,75 +89,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; debug(format("locking path '%1%'") % path); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) { - if (!wait) return false; - throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); - } - lockedPaths->insert(lockPath); - } - - try { - - AutoCloseFD fd; + AutoCloseFD fd; - while (1) { + while (1) { - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - lockedPaths_.lock()->erase(lockPath); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + debug(format("lock acquired on '%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -189,8 +158,6 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on '%1%'") % i.second); @@ -208,11 +175,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index db51f950a..411da0222 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -MakeError(AlreadyLocked, Error); - class PathLocks { private: @@ -37,6 +35,4 @@ public: void setDeletion(bool deletePaths); }; -bool pathIsLockedByMe(const Path & path); - } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5f2219445..5df03da74 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -26,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -36,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 969e4dff3..a344a5ac7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -179,6 +179,36 @@ struct TeeSource : Source } }; +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; /* Convert a function into a sink. */ struct LambdaSink : Sink diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 17aee2d5c..44fa72482 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -84,6 +84,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map<std::string, std::string> newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -1019,10 +1028,22 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); @@ -1047,7 +1068,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fce3cab8d..b538a0b41 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -270,12 +270,14 @@ struct RunOptions std::optional<uid_t> uid; std::optional<uid_t> gid; std::optional<Path> chdir; + std::optional<std::map<std::string, std::string>> environment; Path program; bool searchPath = true; Strings args; std::optional<std::string> input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f324056bb..0cbceb02f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -950,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs) info.sigs = readStrings<StringSet>(in); in >> info.ca; - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, in, NoRepair, NoCheckSigs); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); out << 1; // indicate success diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index b1c1d87de..c0bcfb0c9 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -170,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -228,14 +236,18 @@ public: update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; if (printBuildLogs) { - log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + "> " + ANSI_NORMAL + lastLine); + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { state->activities.erase(i->second); info.lastLine = lastLine; diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 687237add..eca4b2964 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -17,6 +17,7 @@ let { builder = ./dependencies.builder0.sh + "/FOOBAR/../."; input1 = input1 + "/."; input2 = "${input2}/."; + input1_drv = input1; meta.description = "Random test package"; }; diff --git a/tests/function-trace.sh b/tests/function-trace.sh new file mode 100755 index 000000000..f7d93b435 --- /dev/null +++ b/tests/function-trace.sh @@ -0,0 +1,86 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + -vvvv \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh new file mode 100644 index 000000000..1e91282d0 --- /dev/null +++ b/tests/gc-auto.sh @@ -0,0 +1,59 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./tarball.sh) +garbage2=$(nix add-to-store --name garbage2 ./tarball.sh) +garbage3=$(nix add-to-store --name garbage3 ./tarball.sh) + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat <<EOF +with import ./config.nix; mkDerivation { + name = "gc-A"; + buildCommand = '' + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]] + mkdir \$out + echo foo > \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +expr2=$(cat <<EOF +with import ./config.nix; mkDerivation { + name = "gc-B"; + buildCommand = '' + mkdir \$out + echo foo > \$out/bar + echo 1... + sleep 2 + echo 100 > $fake_free + echo 2... + sleep 2 + echo 3... + ''; +} +EOF +) + +nix build -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/tests/local.mk b/tests/local.mk index 1ff68348b..187f96ea2 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,9 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - gc.sh gc-concurrent.sh \ + gc.sh \ + gc-concurrent.sh \ + gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh \ @@ -26,7 +28,9 @@ nix_tests = \ check.sh \ plugins.sh \ search.sh \ - nix-copy-ssh.sh + nix-copy-ssh.sh \ + post-hook.sh \ + function-trace.sh # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) diff --git a/tests/post-hook.sh b/tests/post-hook.sh new file mode 100644 index 000000000..a02657215 --- /dev/null +++ b/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/tests/push-to-store.sh b/tests/push-to-store.sh new file mode 100755 index 000000000..d97eb095d --- /dev/null +++ b/tests/push-to-store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +echo -n "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs |