Merge pull request #9617 from 9999years/stack-overflow-segfault

Fix segfault on infinite recursion in some cases

(cherry picked from commit bf1b294bd81ca76c5ec9fe3ecd52196bf52a8300)
Change-Id: Id137541426ec8536567835953fccf986a3aebf16

6 files changed, 185 insertions, 0 deletions

diff --git a/tests/functional/lang/eval-fail-duplicate-traces.err.exp b/tests/functional/lang/eval-fail-duplicate-traces.err.exp
new file mode 100644
index 000000000..32ad9b376
--- /dev/null
+++ b/tests/functional/lang/eval-fail-duplicate-traces.err.exp
@@ -0,0 +1,44 @@
+error:
+       … from call site
+         at /pwd/lang/eval-fail-duplicate-traces.nix:9:3:
+            8| in
+            9|   throwAfter 2
+             |   ^
+           10|
+
+       … while calling 'throwAfter'
+         at /pwd/lang/eval-fail-duplicate-traces.nix:4:16:
+            3| let
+            4|   throwAfter = n:
+             |                ^
+            5|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-duplicate-traces.nix:6:10:
+            5|     if n > 0
+            6|     then throwAfter (n - 1)
+             |          ^
+            7|     else throw "Uh oh!";
+
+       … while calling 'throwAfter'
+         at /pwd/lang/eval-fail-duplicate-traces.nix:4:16:
+            3| let
+            4|   throwAfter = n:
+             |                ^
+            5|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-duplicate-traces.nix:6:10:
+            5|     if n > 0
+            6|     then throwAfter (n - 1)
+             |          ^
+            7|     else throw "Uh oh!";
+
+       … while calling 'throwAfter'
+         at /pwd/lang/eval-fail-duplicate-traces.nix:4:16:
+            3| let
+            4|   throwAfter = n:
+             |                ^
+            5|     if n > 0
+
+       error: Uh oh!
diff --git a/tests/functional/lang/eval-fail-duplicate-traces.nix b/tests/functional/lang/eval-fail-duplicate-traces.nix
new file mode 100644
index 000000000..17ce374ec
--- /dev/null
+++ b/tests/functional/lang/eval-fail-duplicate-traces.nix
@@ -0,0 +1,9 @@
+# Check that we only omit duplicate stack traces when there's a bunch of them.
+# Here, there's only a couple duplicate entries, so we output them all.
+let
+  throwAfter = n:
+    if n > 0
+    then throwAfter (n - 1)
+    else throw "Uh oh!";
+in
+  throwAfter 2
diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp
new file mode 100644
index 000000000..5d843d827
--- /dev/null
+++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp
@@ -0,0 +1,38 @@
+error:
+       … from call site
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:1:
+            1| (x: x x) (x: x x)
+             | ^
+            2|
+
+       … while calling anonymous lambda
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:2:
+            1| (x: x x) (x: x x)
+             |  ^
+            2|
+
+       … from call site
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:5:
+            1| (x: x x) (x: x x)
+             |     ^
+            2|
+
+       … while calling anonymous lambda
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:11:
+            1| (x: x x) (x: x x)
+             |           ^
+            2|
+
+       … from call site
+         at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+            2|
+
+       (19997 duplicate frames omitted)
+
+       error: stack overflow; max-call-depth exceeded
+       at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14:
+            1| (x: x x) (x: x x)
+             |              ^
+            2|
diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix
new file mode 100644
index 000000000..dd0a8bf2e
--- /dev/null
+++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.nix
@@ -0,0 +1 @@
+(x: x x) (x: x x)
diff --git a/tests/functional/lang/eval-fail-mutual-recursion.err.exp b/tests/functional/lang/eval-fail-mutual-recursion.err.exp
new file mode 100644
index 000000000..dc2e11766
--- /dev/null
+++ b/tests/functional/lang/eval-fail-mutual-recursion.err.exp
@@ -0,0 +1,57 @@
+error:
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:36:3:
+           35| in
+           36|   throwAfterA true 10
+             |   ^
+           37|
+
+       … while calling 'throwAfterA'
+         at /pwd/lang/eval-fail-mutual-recursion.nix:29:26:
+           28|
+           29|   throwAfterA = recurse: n:
+             |                          ^
+           30|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:31:10:
+           30|     if n > 0
+           31|     then throwAfterA recurse (n - 1)
+             |          ^
+           32|     else if recurse
+
+       (19 duplicate frames omitted)
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:33:10:
+           32|     else if recurse
+           33|     then throwAfterB true 10
+             |          ^
+           34|     else throw "Uh oh!";
+
+       … while calling 'throwAfterB'
+         at /pwd/lang/eval-fail-mutual-recursion.nix:22:26:
+           21| let
+           22|   throwAfterB = recurse: n:
+             |                          ^
+           23|     if n > 0
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:24:10:
+           23|     if n > 0
+           24|     then throwAfterB recurse (n - 1)
+             |          ^
+           25|     else if recurse
+
+       (19 duplicate frames omitted)
+
+       … from call site
+         at /pwd/lang/eval-fail-mutual-recursion.nix:26:10:
+           25|     else if recurse
+           26|     then throwAfterA false 10
+             |          ^
+           27|     else throw "Uh oh!";
+
+       (21 duplicate frames omitted)
+
+       error: Uh oh!
diff --git a/tests/functional/lang/eval-fail-mutual-recursion.nix b/tests/functional/lang/eval-fail-mutual-recursion.nix
new file mode 100644
index 000000000..d090d3158
--- /dev/null
+++ b/tests/functional/lang/eval-fail-mutual-recursion.nix
@@ -0,0 +1,36 @@
+# Check that stack frame deduplication only affects consecutive intervals, and
+# that they are reported independently of any preceding sections, even if
+# they're indistinguishable.
+#
+# In terms of the current implementation, we check that we clear the set of
+# "seen frames" after eliding a group of frames.
+#
+# Suppose we have:
+# - 10 frames in a function A
+# - 10 frames in a function B
+# - 10 frames in a function A
+#
+# We want to output:
+# - a few frames of A (skip the rest)
+# - a few frames of B (skip the rest)
+# - a few frames of A (skip the rest)
+#
+# If we implemented this in the naive manner, we'd instead get:
+# - a few frames of A (skip the rest)
+# - a few frames of B (skip the rest, _and_ skip the remaining frames of A)
+let
+  throwAfterB = recurse: n:
+    if n > 0
+    then throwAfterB recurse (n - 1)
+    else if recurse
+    then throwAfterA false 10
+    else throw "Uh oh!";
+
+  throwAfterA = recurse: n:
+    if n > 0
+    then throwAfterA recurse (n - 1)
+    else if recurse
+    then throwAfterB true 10
+    else throw "Uh oh!";
+in
+  throwAfterA true 10