Allow running all the tests with the daemon

When `NIX_DAEMON_PACKAGE` is set, make all the tests use the Nix daemon.
That way we can test every piece of Nix functionality both with and
without the daemon.

Tests for which using the daemon isn’t possible or doesn’t make sens can
selectively be disabled with `needLocalStore`

25 files changed, 103 insertions, 13 deletions

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index 6697ce236..d7bc1507b 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "“--no-require-sigs” can’t be used with the daemon"
+
 # We can produce drvs directly into the binary cache
 clearStore
 clearCacheCache
diff --git a/tests/ca/build-with-garbage-path.sh b/tests/ca/build-with-garbage-path.sh
index e6f878702..9aa08a899 100755
--- a/tests/ca/build-with-garbage-path.sh
+++ b/tests/ca/build-with-garbage-path.sh
@@ -3,7 +3,8 @@
 # Regression test for https://github.com/NixOS/nix/issues/4858
 
 source common.sh
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+requireDaemonNewerThan "2.4pre20210621"
 
 # Get the output path of `rootCA`, and put some garbage instead
 outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)"
diff --git a/tests/ca/common.sh b/tests/ca/common.sh
index e083d873c..c5aa34334 100644
--- a/tests/ca/common.sh
+++ b/tests/ca/common.sh
@@ -1 +1,5 @@
 source ../common.sh
+
+sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+
+restartDaemon
diff --git a/tests/ca/concurrent-builds.sh b/tests/ca/concurrent-builds.sh
index 68441ec76..b442619e2 100755
--- a/tests/ca/concurrent-builds.sh
+++ b/tests/ca/concurrent-builds.sh
@@ -5,7 +5,7 @@
 
 source common.sh
 
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+buggyNeedLocalStore "For some reason, this deadlocks with the daemon"
 
 export NIX_TESTS_CA_BY_DEFAULT=1
 
diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/ca/duplicate-realisation-in-closure.sh
index ca9099641..74c5d25fd 100644
--- a/tests/ca/duplicate-realisation-in-closure.sh
+++ b/tests/ca/duplicate-realisation-in-closure.sh
@@ -1,5 +1,7 @@
 source ./common.sh
 
+requireDaemonNewerThan "2.4pre20210625"
+
 sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
 
 export REMOTE_STORE_DIR="$TEST_ROOT/remote_store"
diff --git a/tests/ca/gc.sh b/tests/ca/gc.sh
index e4f9857d6..e9b6c5ab5 100755
--- a/tests/ca/gc.sh
+++ b/tests/ca/gc.sh
@@ -4,8 +4,6 @@
 
 source common.sh
 
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
-
 export NIX_TESTS_CA_BY_DEFAULT=1
 
 cd ..
diff --git a/tests/ca/post-hook.sh b/tests/ca/post-hook.sh
index 4b8da4cd8..1c9d4f700 100755
--- a/tests/ca/post-hook.sh
+++ b/tests/ca/post-hook.sh
@@ -2,6 +2,8 @@
 
 source common.sh
 
+requireDaemonNewerThan "2.4pre20210626"
+
 sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
 
 export NIX_TESTS_CA_BY_DEFAULT=1
diff --git a/tests/ca/recursive.sh b/tests/ca/recursive.sh
index d9281d91f..648bf0a91 100755
--- a/tests/ca/recursive.sh
+++ b/tests/ca/recursive.sh
@@ -2,6 +2,8 @@
 
 source common.sh
 
+requireDaemonNewerThan "2.4pre20210623"
+
 sed -i 's/experimental-features .*/& ca-derivations ca-references nix-command flakes/' "$NIX_CONF_DIR"/nix.conf
 
 export NIX_TESTS_CA_BY_DEFAULT=1
diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh
index c80feaacf..3d9001bb8 100644
--- a/tests/ca/substitute.sh
+++ b/tests/ca/substitute.sh
@@ -4,7 +4,7 @@
 
 source common.sh
 
-sed -i 's/experimental-features .*/& ca-derivations ca-references/' "$NIX_CONF_DIR"/nix.conf
+needLocalStore "“--no-require-sigs” can’t be used with the daemon"
 
 rm -rf $TEST_ROOT/binary_cache
 
diff --git a/tests/check.sh b/tests/check.sh
index d26d4d8fc..ab48ff865 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -1,5 +1,8 @@
 source common.sh
 
+# XXX: This shouldn’t be, but #4813 cause this test to fail
+buggyNeedLocalStore "see #4813"
+
 checkBuildTempDirRemoved ()
 {
     buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
diff --git a/tests/common.sh.in b/tests/common.sh.in
index d31d3fbb8..bc2e0931f 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -1,5 +1,9 @@
 set -e
 
+if [[ -z "$COMMON_SH_SOURCED" ]]; then
+
+COMMON_SH_SOURCED=1
+
 export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default}
 export NIX_STORE_DIR
 if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
@@ -45,6 +49,9 @@ export busybox="@sandbox_shell@"
 export version=@PACKAGE_VERSION@
 export system=@system@
 
+export IMPURE_VAR1=foo
+export IMPURE_VAR2=bar
+
 cacheDir=$TEST_ROOT/binary-cache
 
 readLink() {
@@ -75,6 +82,10 @@ clearCacheCache() {
 }
 
 startDaemon() {
+    # Don’t start the daemon twice, as this would just make it loop indefinitely
+    if [[ "$NIX_REMOTE" == daemon ]]; then
+      return
+    fi
     # Start the daemon, wait for the socket to appear.  !!!
     # ‘nix-daemon’ should have an option to fork into the background.
     rm -f $NIX_STATE_DIR/daemon-socket/socket
@@ -84,20 +95,44 @@ startDaemon() {
         sleep 1
     done
     pidDaemon=$!
-    trap "kill -9 $pidDaemon" EXIT
+    trap "killDaemon" EXIT
     export NIX_REMOTE=daemon
 }
 
 killDaemon() {
-    kill -9 $pidDaemon
+    kill $pidDaemon
+    for i in {0.10}; do
+        kill -0 $pidDaemon || break
+        sleep 1
+    done
+    kill -9 $pidDaemon || true
     wait $pidDaemon || true
     trap "" EXIT
 }
 
+restartDaemon() {
+  [[ -z "${pidDaemon:-}" ]] && return 0
+
+  killDaemon
+  unset NIX_REMOTE
+  startDaemon
+}
+
 if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi
 
+isDaemonNewer () {
+  [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0
+  local requiredVersion="$1"
+  local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix-daemon --version | cut -d' ' -f3)
+  return [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''2.4''") -ge 0 ]]
+}
+
+requireDaemonNewerThan () {
+  isDaemonNewer "$1" || exit 99
+}
+
 canUseSandbox() {
     if [[ ! $_canUseSandbox ]]; then
         echo "Sandboxing not supported, skipping this test..."
@@ -123,4 +158,22 @@ expect() {
     [[ $res -eq $expected ]]
 }
 
+needLocalStore() {
+  if [[ "$NIX_REMOTE" == "daemon" ]]; then
+    echo "Can’t run through the daemon ($1), skipping this test..."
+    return 99
+  fi
+}
+
+# Just to make it easy to find which tests should be fixed
+buggyNeedLocalStore () {
+  needLocalStore
+}
+
 set -x
+
+if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then
+    startDaemon
+fi
+
+fi # COMMON_SH_SOURCED
diff --git a/tests/db-migration.sh b/tests/db-migration.sh
index e0ff7d311..3f9dc8972 100644
--- a/tests/db-migration.sh
+++ b/tests/db-migration.sh
@@ -3,11 +3,14 @@
 # Only run this if we have an older Nix available
 # XXX: This assumes that the `daemon` package is older than the `client` one
 if [[ -z "$NIX_DAEMON_PACKAGE" ]]; then
-    exit 0
+    exit 99
 fi
 
 source common.sh
 
+killDaemon
+unset NIX_REMOTE
+
 # Fill the db using the older Nix
 PATH_WITH_NEW_NIX="$PATH"
 export PATH="$NIX_DAEMON_PACKAGE/bin:$PATH"
diff --git a/tests/dump-db.sh b/tests/dump-db.sh
index d6eea42aa..48647f403 100644
--- a/tests/dump-db.sh
+++ b/tests/dump-db.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "--dump-db requires a local store"
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)
diff --git a/tests/eval-store.sh b/tests/eval-store.sh
index 7bb86d77c..9ab7a87be 100644
--- a/tests/eval-store.sh
+++ b/tests/eval-store.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+requireDaemonNewerThan "2.4pre20210727"
+
 eval_store=$TEST_ROOT/eval-store
 
 clearStore
diff --git a/tests/fixed.sh b/tests/fixed.sh
index 8f51403a7..90c4c8c32 100644
--- a/tests/fixed.sh
+++ b/tests/fixed.sh
@@ -2,9 +2,6 @@ source common.sh
 
 clearStore
 
-export IMPURE_VAR1=foo
-export IMPURE_VAR2=bar
-
 path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0))
 
 echo 'testing bad...'
diff --git a/tests/gc-auto.sh b/tests/gc-auto.sh
index 6867f2eb4..521d9e539 100644
--- a/tests/gc-auto.sh
+++ b/tests/gc-auto.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "“min-free” and “max-free” are daemon options"
+
 clearStore
 
 garbage1=$(nix store add-path --name garbage1 ./nar-access.sh)
diff --git a/tests/init.sh b/tests/init.sh
index 6e45a939f..3c6d5917d 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -23,6 +23,7 @@ substituters =
 flake-registry = $TEST_ROOT/registry.json
 show-trace = true
 include nix.conf.extra
+trusted-users = $(whoami)
 EOF
 
 cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index eac62d461..3f304ac2f 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon"
+
 clearStore
 
 if ! canUseSandbox; then exit 99; fi
diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh
index de573d4fa..0bca12b42 100644
--- a/tests/multiple-outputs.sh
+++ b/tests/multiple-outputs.sh
@@ -58,7 +58,7 @@ outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-ou
 
 # Delete one of the outputs and rebuild it.  This will cause a hash
 # rewrite.
-nix store delete $TEST_ROOT/result-second --ignore-liveness
+env -u NIX_REMOTE nix store delete $TEST_ROOT/result-second --ignore-liveness
 nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result
 [ "$(cat $TEST_ROOT/result-second/file)" = "second" ]
 [ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ]
diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh
index 61e3df2f9..8c2d05cd5 100644
--- a/tests/optimise-store.sh
+++ b/tests/optimise-store.sh
@@ -26,7 +26,8 @@ if [ "$inode1" = "$inode3" ]; then
     exit 1
 fi
 
-nix-store --optimise
+# XXX: This should work through the daemon too
+NIX_REMOTE="" nix-store --optimise
 
 inode1="$(stat --format=%i $outPath1/foo)"
 inode3="$(stat --format=%i $outPath3/foo)"
diff --git a/tests/post-hook.sh b/tests/post-hook.sh
index 238a8f826..049e40749 100644
--- a/tests/post-hook.sh
+++ b/tests/post-hook.sh
@@ -5,6 +5,9 @@ clearStore
 rm -f $TEST_ROOT/result
 
 export REMOTE_STORE=file:$TEST_ROOT/remote_store
+echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf
+
+restartDaemon
 
 # Build the dependencies and push them to the remote store
 nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh
diff --git a/tests/referrers.sh b/tests/referrers.sh
index 614dd8f5b..81323c280 100644
--- a/tests/referrers.sh
+++ b/tests/referrers.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "uses some low-level store manipulations that aren’t available through the daemon"
+
 clearStore
 
 max=500
diff --git a/tests/repair.sh b/tests/repair.sh
index ba019028d..12dcde8ea 100644
--- a/tests/repair.sh
+++ b/tests/repair.sh
@@ -1,5 +1,7 @@
 source common.sh
 
+needLocalStore "--repair needs a local store"
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)
diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh
index 9612020b8..e585ce37f 100644
--- a/tests/structured-attrs.sh
+++ b/tests/structured-attrs.sh
@@ -1,5 +1,9 @@
 source common.sh
 
+# 27ce722638 required some incompatible changes to the nix file, so skip this
+# tests for the older versions
+requireDaemonNewerThan "2.4pre20210622"
+
 clearStore
 
 rm -f $TEST_ROOT/result
diff --git a/tests/timeout.sh b/tests/timeout.sh
index eea9b5731..e3fb3ebcc 100644
--- a/tests/timeout.sh
+++ b/tests/timeout.sh
@@ -2,6 +2,8 @@
 
 source common.sh
 
+# XXX: This shouldn’t be, but #4813 cause this test to fail
+needLocalStore "see #4813"
 
 set +e
 messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1)