Merge remote-tracking branch 'upstream/master' into derivation-header-include-order

31 files changed, 433 insertions, 65 deletions

diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh
index a3c3c7847..17b63d978 100644
--- a/tests/binary-cache.sh
+++ b/tests/binary-cache.sh
@@ -105,10 +105,24 @@ mv $cacheDir/nar2 $cacheDir/nar
 # incomplete closure.
 clearStore
 
-rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
+rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo)
 
 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
-grep -q "copying path" $TEST_ROOT/log
+grep -q "copying path.*input-0" $TEST_ROOT/log
+grep -q "copying path.*input-2" $TEST_ROOT/log
+grep -q "copying path.*top" $TEST_ROOT/log
+
+
+# Idem, but without cached .narinfo.
+clearStore
+clearCacheCache
+
+nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log
+grep -q "don't know how to build" $TEST_ROOT/log
+grep -q "building.*input-1" $TEST_ROOT/log
+grep -q "building.*input-2" $TEST_ROOT/log
+grep -q "copying path.*input-0" $TEST_ROOT/log
+grep -q "copying path.*top" $TEST_ROOT/log
 
 
 if [ -n "$HAVE_SODIUM" ]; then
diff --git a/tests/build-hook.nix b/tests/build-hook.nix
index 8bff0fe79..8c5ca8cd3 100644
--- a/tests/build-hook.nix
+++ b/tests/build-hook.nix
@@ -4,13 +4,13 @@ let
 
   input1 = mkDerivation {
     name = "build-hook-input-1";
-    builder = ./dependencies.builder1.sh;
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
     requiredSystemFeatures = ["foo"];
   };
 
   input2 = mkDerivation {
     name = "build-hook-input-2";
-    builder = ./dependencies.builder2.sh;
+    buildCommand = "mkdir $out; echo BAR > $out/bar";
   };
 
 in
diff --git a/tests/build-remote.sh b/tests/build-remote.sh
index ddd68f327..a550f4460 100644
--- a/tests/build-remote.sh
+++ b/tests/build-remote.sh
@@ -20,5 +20,5 @@ cat $outPath/foobar | grep FOOBAR
 
 # Ensure that input1 was built on store1 due to the required feature.
 p=$(readlink -f $outPath/input-2)
-(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh)
-nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh
+(! nix path-info --store $TEST_ROOT/store0 --all | grep builder-build-hook-input-1.sh)
+nix path-info --store $TEST_ROOT/store1 --all | grep builder-build-hook-input-1.sh
diff --git a/tests/check.nix b/tests/check.nix
index 56c82e565..bca04fdaf 100644
--- a/tests/check.nix
+++ b/tests/check.nix
@@ -1,12 +1,45 @@
+{checkBuildId ? 0}:
+
 with import ./config.nix;
 
 {
   nondeterministic = mkDerivation {
+    inherit checkBuildId;
     name = "nondeterministic";
     buildCommand =
       ''
         mkdir $out
         date +%s.%N > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  deterministic = mkDerivation {
+    inherit checkBuildId;
+    name = "deterministic";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+      '';
+  };
+
+  failed = mkDerivation {
+    inherit checkBuildId;
+    name = "failed";
+    buildCommand =
+      ''
+        mkdir $out
+        echo date > $out/date
+        echo "CHECK_TMPDIR=$TMPDIR"
+        echo "checkBuildId=$checkBuildId"
+        echo "$checkBuildId" > $TMPDIR/checkBuildId
+        false
       '';
   };
 
diff --git a/tests/check.sh b/tests/check.sh
index bc23a6634..5f25d04cb 100644
--- a/tests/check.sh
+++ b/tests/check.sh
@@ -1,14 +1,57 @@
 source common.sh
 
+checkBuildTempDirRemoved ()
+{
+    buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1)
+    checkBuildIdFile=${buildDir}/checkBuildId
+    [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile
+}
+
+# written to build temp directories to verify created by this instance
+checkBuildId=$(date +%s%N)
+
 clearStore
 
 nix-build dependencies.nix --no-out-link
 nix-build dependencies.nix --no-out-link --check
 
-nix-build check.nix -A nondeterministic --no-out-link
-nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$?
+# check for dangling temporary build directories
+# only retain if build fails and --keep-failed is specified, or...
+# ...build is non-deterministic and --check and --keep-failed are both specified
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \
+    --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$?
+[ "$status" = "100" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log
+if grep -q 'may not be deterministic' $TEST_ROOT/log; then false; fi
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link 2> $TEST_ROOT/log
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check 2> $TEST_ROOT/log || status=$?
+grep 'may not be deterministic' $TEST_ROOT/log
+[ "$status" = "104" ]
+checkBuildTempDirRemoved $TEST_ROOT/log
+
+nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \
+    --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$?
 grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
+if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
 clearStore
 
diff --git a/tests/common.sh.in b/tests/common.sh.in
index 15d7b1ef9..dd7e61822 100644
--- a/tests/common.sh.in
+++ b/tests/common.sh.in
@@ -11,6 +11,7 @@ export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
 export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
 export NIX_STATE_DIR=$TEST_ROOT/var/nix
 export NIX_CONF_DIR=$TEST_ROOT/etc
+unset NIX_USER_CONF_FILES
 export _NIX_TEST_SHARED=$TEST_ROOT/shared
 if [[ -n $NIX_STORE ]]; then
     export _NIX_TEST_NO_SANDBOX=1
@@ -21,6 +22,8 @@ export NIX_REMOTE=$NIX_REMOTE_
 unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
 export HOME=$TEST_HOME
+unset XDG_CONFIG_HOME
+unset XDG_CONFIG_DIRS
 unset XDG_CACHE_HOME
 mkdir -p $TEST_HOME
 
diff --git a/tests/config.nix.in b/tests/config.nix.in
index 0ec2eba6b..a57a8c596 100644
--- a/tests/config.nix.in
+++ b/tests/config.nix.in
@@ -11,7 +11,7 @@ rec {
     derivation ({
       inherit system;
       builder = shell;
-      args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
+      args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")];
       PATH = path;
     } // removeAttrs args ["builder" "meta"])
     // { meta = args.meta or {}; };
diff --git a/tests/config.sh b/tests/config.sh
new file mode 100644
index 000000000..8fa349f11
--- /dev/null
+++ b/tests/config.sh
@@ -0,0 +1,18 @@
+source common.sh
+
+# Test that files are loaded from XDG by default
+export XDG_CONFIG_HOME=/tmp/home
+export XDG_CONFIG_DIRS=/tmp/dir1:/tmp/dir2
+files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
+[[ $files == "/tmp/home/nix/nix.conf:/tmp/dir1/nix/nix.conf:/tmp/dir2/nix/nix.conf" ]]
+
+# Test that setting NIX_USER_CONF_FILES overrides all the default user config files
+export NIX_USER_CONF_FILES=/tmp/file1.conf:/tmp/file2.conf
+files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | xargs)
+[[ $files == "/tmp/file1.conf:/tmp/file2.conf" ]]
+
+# Test that it's possible to load the config from a custom location
+here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")
+export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf
+var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs)
+[[ $var == https://example.com ]]
diff --git a/tests/config/nix-with-substituters.conf b/tests/config/nix-with-substituters.conf
new file mode 100644
index 000000000..90f359a6f
--- /dev/null
+++ b/tests/config/nix-with-substituters.conf
@@ -0,0 +1,2 @@
+experimental-features = nix-command
+substituters = https://example.com
diff --git a/tests/dependencies.builder1.sh b/tests/dependencies.builder1.sh
deleted file mode 100644
index 4b006a17d..000000000
--- a/tests/dependencies.builder1.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-mkdir $out
-echo FOO > $out/foo
diff --git a/tests/dependencies.builder2.sh b/tests/dependencies.builder2.sh
deleted file mode 100644
index 4f886fdb3..000000000
--- a/tests/dependencies.builder2.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-mkdir $out
-echo BAR > $out/bar
diff --git a/tests/dependencies.nix b/tests/dependencies.nix
index eca4b2964..e320d81c9 100644
--- a/tests/dependencies.nix
+++ b/tests/dependencies.nix
@@ -2,18 +2,27 @@ with import ./config.nix;
 
 let {
 
+  input0 = mkDerivation {
+    name = "dependencies-input-0";
+    buildCommand = "mkdir $out; echo foo > $out/bar";
+  };
+
   input1 = mkDerivation {
     name = "dependencies-input-1";
-    builder = ./dependencies.builder1.sh;
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
   };
 
   input2 = mkDerivation {
     name = "dependencies-input-2";
-    builder = "${./dependencies.builder2.sh}";
+    buildCommand = ''
+      mkdir $out
+      echo BAR > $out/bar
+      echo ${input0} > $out/input0
+    '';
   };
 
   body = mkDerivation {
-    name = "dependencies";
+    name = "dependencies-top";
     builder = ./dependencies.builder0.sh + "/FOOBAR/../.";
     input1 = input1 + "/.";
     input2 = "${input2}/.";
diff --git a/tests/dependencies.sh b/tests/dependencies.sh
index 8d0fdc10f..092950aa7 100644
--- a/tests/dependencies.sh
+++ b/tests/dependencies.sh
@@ -6,7 +6,7 @@ drvPath=$(nix-instantiate dependencies.nix)
 
 echo "derivation is $drvPath"
 
-nix-store -q --tree "$drvPath" | grep '───.*builder1.sh'
+nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh'
 
 # Test Graphviz graph generation.
 nix-store -q --graph "$drvPath" > $TEST_ROOT/graph
diff --git a/tests/export-graph.sh b/tests/export-graph.sh
index a6fd69054..a1449b34e 100644
--- a/tests/export-graph.sh
+++ b/tests/export-graph.sh
@@ -11,7 +11,7 @@ checkRef() {
 
 outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result)
 
-test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 2 || fail "bad nr of references"
+test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 3 || fail "bad nr of references"
 
 checkRef input-2
 for i in $(cat $outPath); do checkRef $i; done
diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh
index ed8fa14d6..d9c9874f5 100644
--- a/tests/fetchGit.sh
+++ b/tests/fetchGit.sh
@@ -11,7 +11,7 @@ repo=$TEST_ROOT/git
 
 export _NIX_FORCE_HTTP=1
 
-rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2
+rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/shallow
 
 git init $repo
 git -C $repo config user.email "foobar@example.com"
@@ -25,8 +25,16 @@ rev1=$(git -C $repo rev-parse HEAD)
 
 echo world > $repo/hello
 git -C $repo commit -m 'Bla2' -a
+git -C $repo worktree add $TEST_ROOT/worktree
+echo hello >> $TEST_ROOT/worktree/hello
 rev2=$(git -C $repo rev-parse HEAD)
 
+# Fetch a worktree
+unset _NIX_FORCE_HTTP
+path0=$(nix eval --raw "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath")
+export _NIX_FORCE_HTTP=1
+[[ $(tail -n 1 $path0/hello) = "hello" ]]
+
 # Fetch the default branch.
 path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(cat $path/hello) = world ]]
@@ -50,9 +58,6 @@ path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 [[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]]
 [[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]]
 
-# But with TTL 0, it should fail.
-(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv)
-
 # Fetching with a explicit hash should succeed.
 path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath")
 [[ $path = $path2 ]]
@@ -74,6 +79,7 @@ echo bar > $repo/dir2/bar
 git -C $repo add dir1/foo
 git -C $repo rm hello
 
+unset _NIX_FORCE_HTTP
 path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
 [ ! -e $path2/hello ]
 [ ! -e $path2/bar ]
@@ -110,9 +116,9 @@ path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
 git -C $repo checkout $rev2 -b dev
 echo dev > $repo/hello
 
-# File URI uses 'master' unless specified otherwise
+# File URI uses dirty tree unless specified otherwise
 path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath")
-[[ $path = $path2 ]]
+[ $(cat $path2/hello) = dev ]
 
 # Using local path with branch other than 'master' should work when clean or dirty
 path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath")
@@ -131,9 +137,9 @@ path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outP
 
 
 # Nuke the cache
-rm -rf $TEST_HOME/.cache/nix/gitv2
+rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH
+# Try again, but without 'git' on PATH. This should fail.
 NIX=$(command -v nix)
 # This should fail
 (! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
@@ -141,3 +147,13 @@ NIX=$(command -v nix)
 # Try again, with 'git' available.  This should work.
 path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
+
+# Fetching a shallow repo shouldn't work by default, because we can't
+# return a revCount.
+git clone --depth 1 file://$repo $TEST_ROOT/shallow
+(! nix eval --raw "(builtins.fetchGit { url = $TEST_ROOT/shallow; ref = \"dev\"; }).outPath")
+
+# But you can request a shallow clone, which won't return a revCount.
+path6=$(nix eval --raw "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).outPath")
+[[ $path3 = $path6 ]]
+[[ $(nix eval "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]]
diff --git a/tests/fetchGitRefs.sh b/tests/fetchGitRefs.sh
new file mode 100644
index 000000000..23934698e
--- /dev/null
+++ b/tests/fetchGitRefs.sh
@@ -0,0 +1,111 @@
+source common.sh
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git tests"
+    exit 99
+fi
+
+clearStore
+
+repo="$TEST_ROOT/git"
+
+rm -rf "$repo" "${repo}-tmp" "$TEST_HOME/.cache/nix"
+
+git init "$repo"
+git -C "$repo" config user.email "foobar@example.com"
+git -C "$repo" config user.name "Foobar"
+
+echo utrecht > "$repo"/hello
+git -C "$repo" add hello
+git -C "$repo" commit -m 'Bla1'
+
+path=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath")
+
+# Test various combinations of ref names
+# (taken from the git project)
+
+# git help check-ref-format
+#       Git imposes the following rules on how references are named:
+#
+#        1. They can include slash / for hierarchical (directory) grouping, but no slash-separated component can begin with a dot .  or end with the sequence .lock.
+#        2. They must contain at least one /. This enforces the presence of a category like heads/, tags/ etc. but the actual names are not restricted. If the --allow-onelevel option is used, this rule is waived.
+#        3. They cannot have two consecutive dots ..  anywhere.
+#        4. They cannot have ASCII control characters (i.e. bytes whose values are lower than \040, or \177 DEL), space, tilde ~, caret ^, or colon : anywhere.
+#        5. They cannot have question-mark ?, asterisk *, or open bracket [ anywhere. See the --refspec-pattern option below for an exception to this rule.
+#        6. They cannot begin or end with a slash / or contain multiple consecutive slashes (see the --normalize option below for an exception to this rule)
+#        7. They cannot end with a dot ..
+#        8. They cannot contain a sequence @{.
+#        9. They cannot be the single character @.
+#       10. They cannot contain a \.
+
+valid_ref() {
+    { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
+    git check-ref-format --branch "$1" >/dev/null
+    git -C "$repo" branch "$1" master >/dev/null
+    path1=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath")
+    [[ $path1 = $path ]]
+    git -C "$repo" branch -D "$1" >/dev/null
+}
+
+invalid_ref() {
+    { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; }
+    # special case for a sole @:
+    # --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel
+    if [ "$1" = "@" ]; then
+        (! git check-ref-format --allow-onelevel "$1" >/dev/null 2>&1)
+    else
+        (! git check-ref-format --branch "$1" >/dev/null 2>&1)
+    fi
+    nix --debug eval --raw "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath" 2>&1 | grep 'invalid Git branch/tag name' >/dev/null
+}
+
+
+valid_ref 'foox'
+valid_ref '1337'
+valid_ref 'foo.baz'
+valid_ref 'foo/bar/baz'
+valid_ref 'foo./bar'
+valid_ref 'heads/foo@bar'
+valid_ref "$(printf 'heads/fu\303\237')"
+valid_ref 'foo-bar-baz'
+valid_ref '$1'
+valid_ref 'foo.locke'
+
+invalid_ref 'refs///heads/foo'
+invalid_ref 'heads/foo/'
+invalid_ref '///heads/foo'
+invalid_ref '.foo'
+invalid_ref './foo'
+invalid_ref './foo/bar'
+invalid_ref 'foo/./bar'
+invalid_ref 'foo/bar/.'
+invalid_ref 'foo bar'
+invalid_ref 'foo?bar'
+invalid_ref 'foo^bar'
+invalid_ref 'foo~bar'
+invalid_ref 'foo:bar'
+invalid_ref 'foo[bar'
+invalid_ref 'foo/bar/.'
+invalid_ref '.refs/foo'
+invalid_ref 'refs/heads/foo.'
+invalid_ref 'heads/foo..bar'
+invalid_ref 'heads/foo?bar'
+invalid_ref 'heads/foo.lock'
+invalid_ref 'heads///foo.lock'
+invalid_ref 'foo.lock/bar'
+invalid_ref 'foo.lock///bar'
+invalid_ref 'heads/v@{ation'
+invalid_ref 'heads/foo\.ar' # should fail due to \
+invalid_ref 'heads/foo\bar' # should fail due to \
+invalid_ref "$(printf 'heads/foo\t')" # should fail because it has a TAB
+invalid_ref "$(printf 'heads/foo\177')"
+invalid_ref '@'
+
+invalid_ref 'foo/*'
+invalid_ref '*/foo'
+invalid_ref 'foo/*/bar'
+invalid_ref '*'
+invalid_ref 'foo/*/*'
+invalid_ref '*/foo/*'
+invalid_ref '/foo'
+invalid_ref ''
diff --git a/tests/fetchGitSubmodules.sh b/tests/fetchGitSubmodules.sh
new file mode 100644
index 000000000..4c2c13f1a
--- /dev/null
+++ b/tests/fetchGitSubmodules.sh
@@ -0,0 +1,97 @@
+source common.sh
+
+set -u
+
+if [[ -z $(type -p git) ]]; then
+    echo "Git not installed; skipping Git submodule tests"
+    exit 99
+fi
+
+clearStore
+
+rootRepo=$TEST_ROOT/gitSubmodulesRoot
+subRepo=$TEST_ROOT/gitSubmodulesSub
+
+rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix
+
+initGitRepo() {
+    git init $1
+    git -C $1 config user.email "foobar@example.com"
+    git -C $1 config user.name "Foobar"
+}
+
+addGitContent() {
+    echo "lorem ipsum" > $1/content
+    git -C $1 add content
+    git -C $1 commit -m "Initial commit"
+}
+
+initGitRepo $subRepo
+addGitContent $subRepo
+
+initGitRepo $rootRepo
+
+git -C $rootRepo submodule init
+git -C $rootRepo submodule add $subRepo sub
+git -C $rootRepo add sub
+git -C $rootRepo commit -m "Add submodule"
+
+rev=$(git -C $rootRepo rev-parse HEAD)
+
+r1=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+r2=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath")
+r3=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r2 ]]
+[[ $r2 != $r3 ]]
+
+r4=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath")
+r5=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath")
+r6=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r7=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+r8=$(nix eval --raw "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+
+[[ $r1 == $r4 ]]
+[[ $r4 == $r5 ]]
+[[ $r3 == $r6 ]]
+[[ $r6 == $r7 ]]
+[[ $r7 == $r8 ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = false; }).submodules")
+[[ $have_submodules == false ]]
+
+have_submodules=$(nix eval "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).submodules")
+[[ $have_submodules == true ]]
+
+pathWithoutSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath")
+pathWithSubmodules=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgain=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath")
+pathWithSubmodulesAgainWithRef=$(nix eval --raw "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath")
+
+# The resulting store path cannot be the same.
+[[ $pathWithoutSubmodules != $pathWithSubmodules ]]
+
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]]
+
+# Checking out the same repo with submodules returns in the same store path.
+[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]]
+
+# The submodules flag is actually honored.
+[[ ! -e $pathWithoutSubmodules/sub/content ]]
+[[ -e $pathWithSubmodules/sub/content ]]
+
+[[ -e $pathWithSubmodulesAgainWithRef/sub/content ]]
+
+# No .git directory or submodule reference files must be left
+test "$(find "$pathWithSubmodules" -name .git)" = ""
+
+# Git repos without submodules can be fetched with submodules = true.
+subRev=$(git -C $subRepo rev-parse HEAD)
+noSubmoduleRepoBaseline=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath")
+noSubmoduleRepo=$(nix eval --raw "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath")
+
+[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]]
diff --git a/tests/gc-concurrent.nix b/tests/gc-concurrent.nix
index c0595cc47..21671ea2c 100644
--- a/tests/gc-concurrent.nix
+++ b/tests/gc-concurrent.nix
@@ -4,12 +4,12 @@ rec {
 
   input1 = mkDerivation {
     name = "dependencies-input-1";
-    builder = ./dependencies.builder1.sh;
+    buildCommand = "mkdir $out; echo FOO > $out/foo";
   };
 
   input2 = mkDerivation {
     name = "dependencies-input-2";
-    builder = ./dependencies.builder2.sh;
+    buildCommand = "mkdir $out; echo BAR > $out/bar";
   };
 
   test1 = mkDerivation {
@@ -23,5 +23,5 @@ rec {
     builder = ./gc-concurrent2.builder.sh;
     inherit input1 input2;
   };
-  
+
 }
diff --git a/tests/init.sh b/tests/init.sh
index 6a119aad0..c62c4856a 100644
--- a/tests/init.sh
+++ b/tests/init.sh
@@ -17,7 +17,7 @@ cat > "$NIX_CONF_DIR"/nix.conf <<EOF
 build-users-group =
 keep-derivations = false
 sandbox = false
-experimental-features = nix-command
+experimental-features = nix-command flakes
 include nix.conf.extra
 EOF
 
diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/lang/eval-okay-getattrpos-functionargs.exp
new file mode 100644
index 000000000..7f9ac40e8
--- /dev/null
+++ b/tests/lang/eval-okay-getattrpos-functionargs.exp
@@ -0,0 +1 @@
+{ column = 11; file = "eval-okay-getattrpos-functionargs.nix"; line = 2; }
diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/lang/eval-okay-getattrpos-functionargs.nix
new file mode 100644
index 000000000..11d6bb0e3
--- /dev/null
+++ b/tests/lang/eval-okay-getattrpos-functionargs.nix
@@ -0,0 +1,4 @@
+let
+  fun = { foo }: {};
+  pos = builtins.unsafeGetAttrPos "foo" (builtins.functionArgs fun);
+in { inherit (pos) column line; file = baseNameOf pos.file; }
diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh
index 52967d07d..16abd974c 100644
--- a/tests/linux-sandbox.sh
+++ b/tests/linux-sandbox.sh
@@ -28,3 +28,10 @@ nix cat-store $outPath/foobar | grep FOOBAR
 
 # Test --check without hash rewriting.
 nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store
+
+# Test that sandboxed builds with --check and -K can move .check directory to store
+nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link
+
+(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log)
+if grep -q 'error: renaming' $TEST_ROOT/log; then false; fi
+grep -q 'may not be deterministic' $TEST_ROOT/log
diff --git a/tests/local.mk b/tests/local.mk
index dab3a23b6..536661af8 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -1,5 +1,6 @@
 nix_tests = \
   init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
+  config.sh \
   gc.sh \
   gc-concurrent.sh \
   gc-auto.sh \
@@ -17,9 +18,11 @@ nix_tests = \
   nar-access.sh \
   structured-attrs.sh \
   fetchGit.sh \
+  fetchGitRefs.sh \
+  fetchGitSubmodules.sh \
   fetchMercurial.sh \
   signing.sh \
-  run.sh \
+  shell.sh \
   brotli.sh \
   pure-eval.sh \
   check.sh \
diff --git a/tests/misc.sh b/tests/misc.sh
index eda016416..fd4908e25 100644
--- a/tests/misc.sh
+++ b/tests/misc.sh
@@ -16,4 +16,6 @@ nix-env --foo 2>&1 | grep "no operation"
 nix-env -q --foo 2>&1 | grep "unknown flag"
 
 # Eval Errors.
-nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$"
+eval_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true)
+echo $eval_res | grep "(string) (1:15)"
+echo $eval_res | grep "infinite recursion encountered"
diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh
index 93f837bef..49c68981a 100644
--- a/tests/nix-channel.sh
+++ b/tests/nix-channel.sh
@@ -32,10 +32,10 @@ if [ "$xmllint" != false ]; then
     $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
 fi
 grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
-grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml
 
 # Do an install.
-nix-env -i dependencies
+nix-env -i dependencies-top
 [ -e $TEST_HOME/.nix-profile/foobar ]
 
 clearProfiles
@@ -51,9 +51,9 @@ if [ "$xmllint" != false ]; then
     $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML"
 fi
 grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml
-grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml
+grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml
 
 # Do an install.
-nix-env -i dependencies
+nix-env -i dependencies-top
 [ -e $TEST_HOME/.nix-profile/foobar ]
 
diff --git a/tests/plugins/local.mk b/tests/plugins/local.mk
index 1d2bac052..82ad99402 100644
--- a/tests/plugins/local.mk
+++ b/tests/plugins/local.mk
@@ -7,3 +7,5 @@ libplugintest_SOURCES := $(d)/plugintest.cc
 libplugintest_ALLOW_UNDEFINED := 1
 
 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1
+
+libplugintest_CXXFLAGS := -I src/libutil -I src/libexpr
diff --git a/tests/remote-store.sh b/tests/remote-store.sh
index 77437658e..4cc73465a 100644
--- a/tests/remote-store.sh
+++ b/tests/remote-store.sh
@@ -4,7 +4,7 @@ clearStore
 
 startDaemon
 
-storeCleared=1 $SHELL ./user-envs.sh
+storeCleared=1 NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs.sh
 
 nix-store --dump-db > $TEST_ROOT/d1
 NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2
diff --git a/tests/run.sh b/tests/run.sh
deleted file mode 100644
index d1dbfd6bd..000000000
--- a/tests/run.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-source common.sh
-
-clearStore
-clearCache
-
-nix run -f run.nix hello -c hello | grep 'Hello World'
-nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS'
-
-if ! canUseSandbox; then exit; fi
-
-chmod -R u+w $TEST_ROOT/store0 || true
-rm -rf $TEST_ROOT/store0
-
-clearStore
-
-path=$(nix eval --raw -f run.nix hello)
-
-# Note: we need the sandbox paths to ensure that the shell is
-# visible in the sandbox.
-nix run --sandbox-build-dir /build-tmp \
-    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
-    --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World'
-
-path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello')
-
-[[ $path/bin/hello = $path2 ]]
-
-[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]
diff --git a/tests/run.nix b/tests/shell-hello.nix
index 77dcbd2a9..77dcbd2a9 100644
--- a/tests/run.nix
+++ b/tests/shell-hello.nix
diff --git a/tests/shell.sh b/tests/shell.sh
new file mode 100644
index 000000000..7a9ee8ab0
--- /dev/null
+++ b/tests/shell.sh
@@ -0,0 +1,28 @@
+source common.sh
+
+clearStore
+clearCache
+
+nix shell -f shell-hello.nix hello -c hello | grep 'Hello World'
+nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS'
+
+if ! canUseSandbox; then exit; fi
+
+chmod -R u+w $TEST_ROOT/store0 || true
+rm -rf $TEST_ROOT/store0
+
+clearStore
+
+path=$(nix eval --raw -f shell-hello.nix hello)
+
+# Note: we need the sandbox paths to ensure that the shell is
+# visible in the sandbox.
+nix shell --sandbox-build-dir /build-tmp \
+    --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \
+    --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World'
+
+path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello')
+
+[[ $path/bin/hello = $path2 ]]
+
+[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]]
diff --git a/tests/tarball.sh b/tests/tarball.sh
index 8adb8d72f..b3ec16d40 100644
--- a/tests/tarball.sh
+++ b/tests/tarball.sh
@@ -10,6 +10,8 @@ mkdir -p $tarroot
 cp dependencies.nix $tarroot/default.nix
 cp config.nix dependencies.builder*.sh $tarroot/
 
+hash=$(nix hash-path $tarroot)
+
 test_tarball() {
     local ext="$1"
     local compressor="$2"
@@ -25,6 +27,11 @@ test_tarball() {
 
     nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)"
 
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })"
+    nix-build --experimental-features flakes -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" 2>&1 | grep 'NAR hash mismatch in input'
+
     nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext
     nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball$ext
     (! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball$ext)