Implement a repl characterization test system

This allows for automating using the repl without needing a PTY, with
very easy to write test files.

Change-Id: Ia8d7854edd91f93477638942cb6fc261354e6035

4 files changed, 338 insertions, 0 deletions

diff --git a/tests/functional/repl_characterization/data/basic_repl.test b/tests/functional/repl_characterization/data/basic_repl.test
new file mode 100644
index 000000000..a8dea6d7c
--- /dev/null
+++ b/tests/functional/repl_characterization/data/basic_repl.test
@@ -0,0 +1,60 @@
+  nix-repl> 1 + 1
+  2
+
+  nix-repl> :doc builtins.head
+  Synopsis: builtins.head list
+
+      Return the first element of a list; abort evaluation if
+      the argument isn’t a list or is an empty list. You can
+      test whether a list is empty by comparing it with [].
+
+  nix-repl> f = a: "" + a
+
+Expect the trace to not contain any traceback:
+
+  nix-repl> f 2
+  error:
+         … while evaluating a path segment
+           at «string»:1:10:
+              1|  a: "" + a
+               |          ^
+
+         error: cannot coerce an integer to a string: 2
+
+  nix-repl> :te
+  showing error traces
+
+Expect the trace to have traceback:
+
+  nix-repl> f 2
+  error:
+         … from call site
+           at «string»:1:1:
+              1| f 2
+               | ^
+
+         … while calling anonymous lambda
+           at «string»:1:2:
+              1|  a: "" + a
+               |  ^
+
+         … while evaluating a path segment
+           at «string»:1:10:
+              1|  a: "" + a
+               |          ^
+
+         error: cannot coerce an integer to a string: 2
+
+Turning it off should also work:
+
+  nix-repl> :te
+  not showing error traces
+
+  nix-repl> f 2
+  error:
+         … while evaluating a path segment
+           at «string»:1:10:
+              1|  a: "" + a
+               |          ^
+
+         error: cannot coerce an integer to a string: 2
diff --git a/tests/functional/repl_characterization/repl_characterization.cc b/tests/functional/repl_characterization/repl_characterization.cc
index 5b73e7a89..993c30232 100644
--- a/tests/functional/repl_characterization/repl_characterization.cc
+++ b/tests/functional/repl_characterization/repl_characterization.cc
@@ -5,8 +5,11 @@
 #include <optional>
 #include <unistd.h>
 
+#include "test-session.hh"
+#include "util.hh"
 #include "tests/characterization.hh"
 #include "tests/cli-literate-parser.hh"
+#include "tests/terminal-code-eater.hh"
 
 using namespace std::string_literals;
 
@@ -14,6 +17,18 @@ namespace nix {
 
 static constexpr const char * REPL_PROMPT = "nix-repl> ";
 
+// ASCII ENQ character
+static constexpr const char * AUTOMATION_PROMPT = "\x05";
+
+static std::string_view trimOutLog(std::string_view outLog)
+{
+    const std::string trailer = "\n"s + AUTOMATION_PROMPT;
+    if (outLog.ends_with(trailer)) {
+        outLog.remove_suffix(trailer.length());
+    }
+    return outLog;
+}
+
 class ReplSessionTest : public CharacterizationTest
 {
     Path unitTestData = getUnitTestData();
@@ -23,6 +38,43 @@ public:
     {
         return unitTestData + "/" + testStem;
     }
+
+    void runReplTest(std::string_view const & content, std::vector<std::string> extraArgs = {}) const
+    {
+        auto syntax = CLILiterateParser::parse(REPL_PROMPT, content);
+
+        // FIXME: why does this need two --quiets
+        // show-trace is on by default due to test configuration, but is not a standard
+        Strings args{"--quiet", "repl", "--quiet", "--option", "show-trace", "false", "--offline", "--extra-experimental-features", "repl-automation"};
+        args.insert(args.end(), extraArgs.begin(), extraArgs.end());
+
+        auto nixBin = canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR));
+
+        auto process = RunningProcess::start(nixBin + "/nix", args);
+        auto session = TestSession{AUTOMATION_PROMPT, std::move(process)};
+
+        for (auto & bit : syntax) {
+            if (bit.kind != CLILiterateParser::NodeKind::COMMAND) {
+                continue;
+            }
+
+            if (!session.waitForPrompt()) {
+                ASSERT_TRUE(false);
+            }
+            session.runCommand(bit.text);
+        }
+        if (!session.waitForPrompt()) {
+            ASSERT_TRUE(false);
+        }
+        session.close();
+
+        auto parsedOutLog = CLILiterateParser::parse(AUTOMATION_PROMPT, trimOutLog(session.outLog), 0);
+
+        parsedOutLog = CLILiterateParser::tidyOutputForComparison(std::move(parsedOutLog));
+        syntax = CLILiterateParser::tidyOutputForComparison(std::move(syntax));
+
+        ASSERT_EQ(parsedOutLog, syntax);
+    }
 };
 
 TEST_F(ReplSessionTest, parses)
@@ -39,4 +91,10 @@ TEST_F(ReplSessionTest, parses)
         return out.str();
     });
 }
+
+TEST_F(ReplSessionTest, repl_basic)
+{
+    readTest("basic_repl.test", [this](std::string input) { runReplTest(input); });
+}
+
 };
diff --git a/tests/functional/repl_characterization/test-session.cc b/tests/functional/repl_characterization/test-session.cc
new file mode 100644
index 000000000..c35030fc7
--- /dev/null
+++ b/tests/functional/repl_characterization/test-session.cc
@@ -0,0 +1,151 @@
+#include <iostream>
+#include <unistd.h>
+
+#include "test-session.hh"
+#include "util.hh"
+#include "tests/debug-char.hh"
+
+namespace nix {
+
+static constexpr const bool DEBUG_REPL_PARSER = false;
+
+RunningProcess RunningProcess::start(std::string executable, Strings args)
+{
+    args.push_front(executable);
+
+    Pipe procStdin{};
+    Pipe procStdout{};
+
+    procStdin.create();
+    procStdout.create();
+
+    // This is separate from runProgram2 because we have different IO requirements
+    pid_t pid = startProcess([&]() {
+        if (dup2(procStdout.writeSide.get(), STDOUT_FILENO) == -1)
+            throw SysError("dupping stdout");
+        if (dup2(procStdin.readSide.get(), STDIN_FILENO) == -1)
+            throw SysError("dupping stdin");
+        procStdin.writeSide.close();
+        procStdout.readSide.close();
+        if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1)
+            throw SysError("dupping stderr");
+        execv(executable.c_str(), stringsToCharPtrs(args).data());
+        throw SysError("exec did not happen");
+    });
+
+    procStdout.writeSide.close();
+    procStdin.readSide.close();
+
+    return RunningProcess{
+        .pid = pid,
+        .procStdin = std::move(procStdin),
+        .procStdout = std::move(procStdout),
+    };
+}
+
+[[gnu::unused]]
+std::ostream & operator<<(std::ostream & os, ReplOutputParser::State s)
+{
+    switch (s) {
+    case ReplOutputParser::State::Prompt:
+        os << "prompt";
+        break;
+    case ReplOutputParser::State::Context:
+        os << "context";
+        break;
+    }
+    return os;
+}
+
+void ReplOutputParser::transition(State new_state, char responsible_char, bool wasPrompt)
+{
+    if constexpr (DEBUG_REPL_PARSER) {
+        std::cerr << "transition " << new_state << " for " << DebugChar{responsible_char}
+                  << (wasPrompt ? " [prompt]" : "") << "\n";
+    }
+    state = new_state;
+    pos_in_prompt = 0;
+}
+
+bool ReplOutputParser::feed(char c)
+{
+    if (c == '\n') {
+        transition(State::Prompt, c);
+        return false;
+    }
+    switch (state) {
+    case State::Context:
+        break;
+    case State::Prompt:
+        if (pos_in_prompt == prompt.length() - 1 && prompt[pos_in_prompt] == c) {
+            transition(State::Context, c, true);
+            return true;
+        }
+        if (pos_in_prompt >= prompt.length() - 1 || prompt[pos_in_prompt] != c) {
+            transition(State::Context, c);
+            break;
+        }
+        pos_in_prompt++;
+        break;
+    }
+    return false;
+}
+
+/** Waits for the prompt and then returns if a prompt was found */
+bool TestSession::waitForPrompt()
+{
+    std::vector<char> buf(1024);
+
+    for (;;) {
+        ssize_t res = read(proc.procStdout.readSide.get(), buf.data(), buf.size());
+
+        if (res < 0) {
+            throw SysError("read");
+        }
+        if (res == 0) {
+            return false;
+        }
+
+        bool foundPrompt = false;
+        for (ssize_t i = 0; i < res; ++i) {
+            // foundPrompt = foundPrompt || outputParser.feed(buf[i]);
+            bool wasEaten = true;
+            eater.feed(buf[i], [&](char c) {
+                wasEaten = false;
+                foundPrompt = outputParser.feed(buf[i]) || foundPrompt;
+
+                outLog.push_back(c);
+            });
+
+            if constexpr (DEBUG_REPL_PARSER) {
+                std::cerr << "raw " << DebugChar{buf[i]} << (wasEaten ? " [eaten]" : "") << "\n";
+            }
+        }
+
+        if (foundPrompt) {
+            return true;
+        }
+    }
+}
+
+void TestSession::close()
+{
+    proc.procStdin.close();
+    proc.procStdout.close();
+}
+
+void TestSession::runCommand(std::string command)
+{
+    if constexpr (DEBUG_REPL_PARSER)
+        std::cerr << "runCommand " << command << "\n";
+    command += "\n";
+    // We have to feed a newline into the output parser, since Nix might not
+    // give us a newline before a prompt in all cases (it might clear line
+    // first, e.g.)
+    outputParser.feed('\n');
+    // Echo is disabled, so we have to make our own
+    outLog.append(command);
+    writeFull(proc.procStdin.writeSide.get(), command, false);
+}
+
+};
diff --git a/tests/functional/repl_characterization/test-session.hh b/tests/functional/repl_characterization/test-session.hh
new file mode 100644
index 000000000..19636640b
--- /dev/null
+++ b/tests/functional/repl_characterization/test-session.hh
@@ -0,0 +1,69 @@
+#pragma once
+///@file
+
+#include <sched.h>
+#include <string>
+
+#include "util.hh"
+#include "tests/terminal-code-eater.hh"
+
+namespace nix {
+
+struct RunningProcess
+{
+    pid_t pid;
+    Pipe procStdin;
+    Pipe procStdout;
+
+    static RunningProcess start(std::string executable, Strings args);
+};
+
+/** DFA that catches repl prompts */
+class ReplOutputParser
+{
+public:
+    ReplOutputParser(std::string prompt)
+        : prompt(prompt)
+    {
+        assert(!prompt.empty());
+    }
+    /** Feeds in a character and returns whether this is an open prompt */
+    bool feed(char c);
+
+    enum class State {
+        Prompt,
+        Context,
+    };
+
+private:
+    State state = State::Prompt;
+    size_t pos_in_prompt = 0;
+    std::string const prompt;
+
+    void transition(State state, char responsible_char, bool wasPrompt = false);
+};
+
+struct TestSession
+{
+    RunningProcess proc;
+    ReplOutputParser outputParser;
+    TerminalCodeEater eater;
+    std::string outLog;
+    std::string prompt;
+
+    TestSession(std::string prompt, RunningProcess && proc)
+        : proc(std::move(proc))
+        , outputParser(prompt)
+        , eater{}
+        , outLog{}
+        , prompt(prompt)
+    {
+    }
+
+    bool waitForPrompt();
+
+    void runCommand(std::string command);
+
+    void close();
+};
+};