fix: check if it is a Real terminal, not just if it is a terminal

This will stop printing stuff to dumb terminals that they don't support.

I've overall audited usage of isatty and replaced the ones with intent
to mean "is a Real terminal" with checking for that. I've also caught a
case of carelessly assuming "is a tty" means "should be colour" in
nix-env.

Change-Id: I6d83725d9a2d932ac94ff2294f92c0a1100d23c9

3 files changed, 42 insertions, 7 deletions

diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index cbeb7aa36..7d9482814 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -37,7 +37,15 @@ void Logger::warn(const std::string & msg)
 
 void Logger::writeToStdout(std::string_view s)
 {
-    writeFull(STDOUT_FILENO, filterANSIEscapes(s, !shouldANSI(), std::numeric_limits<unsigned int>::max(), false));
+    writeFull(
+        STDOUT_FILENO,
+        filterANSIEscapes(
+            s,
+            !shouldANSI(StandardOutputStream::Stdout),
+            std::numeric_limits<unsigned int>::max(),
+            false
+        )
+    );
     writeFull(STDOUT_FILENO, "\n");
 }
 
diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc
index 2ba1cb81b..25e97e599 100644
--- a/src/libutil/terminal.cc
+++ b/src/libutil/terminal.cc
@@ -7,7 +7,12 @@
 
 namespace nix {
 
-bool shouldANSI()
+bool isOutputARealTerminal(StandardOutputStream fileno)
+{
+    return isatty(int(fileno)) && getEnv("TERM").value_or("dumb") != "dumb";
+}
+
+bool shouldANSI(StandardOutputStream fileno)
 {
     // Implements the behaviour described by https://bixense.com/clicolors/
     // As well as https://force-color.org/ for compatibility, since it fits in the same shape.
@@ -16,14 +21,14 @@ bool shouldANSI()
     // unset    x        set            Yes
     // unset    x        unset          If attached to a terminal
     //                                  [we choose the "modern" approach of colour-by-default]
-    auto compute = []() -> bool {
+    auto compute = [](StandardOutputStream fileno) -> bool {
         bool mustNotColour = getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value();
         bool shouldForce = getEnv("CLICOLOR_FORCE").has_value() || getEnv("FORCE_COLOR").has_value();
-        bool isTerminal = isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb";
+        bool isTerminal = isOutputARealTerminal(fileno);
         return !mustNotColour && (shouldForce || isTerminal);
     };
-    static bool cached = compute();
-    return cached;
+    static bool cached[2] = {compute(StandardOutputStream::Stdout), compute(StandardOutputStream::Stderr)};
+    return cached[int(fileno) - 1];
 }
 
 // FIXME(jade): replace with TerminalCodeEater. wowie this is evil code.
diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh
index 2c422ecff..28c96c780 100644
--- a/src/libutil/terminal.hh
+++ b/src/libutil/terminal.hh
@@ -6,6 +6,26 @@
 
 namespace nix {
 
+enum class StandardOutputStream {
+    Stdout = 1,
+    Stderr = 2,
+};
+
+/**
+ * Determine whether the output is a real terminal (i.e. not dumb, not a pipe).
+ *
+ * This is probably not what you want, you may want shouldANSI() or something
+ * more specific. Think about how the output should work with a pager or
+ * entirely non-interactive scripting use.
+ *
+ * The user may be redirecting the Lix output to a pager, but have stderr
+ * connected to a terminal. Think about where you are outputting the text when
+ * deciding whether to use STDERR_FILENO or STDOUT_FILENO.
+ *
+ * \param fileno file descriptor number to check if it is a tty
+ */
+bool isOutputARealTerminal(StandardOutputStream fileno);
+
 /**
  * Determine whether ANSI escape sequences are appropriate for the
  * present output.
@@ -18,8 +38,10 @@ namespace nix {
  * - CLICOLOR_FORCE or FORCE_COLOR set   -> enable colour
  * - The output is a tty; TERM != "dumb" -> enable colour
  * - Otherwise                           -> disable colour
+ *
+ * \param fileno which file descriptor number to consider. Use the one you are outputting to
  */
-bool shouldANSI();
+bool shouldANSI(StandardOutputStream fileno = StandardOutputStream::Stderr);
 
 /**
  * Truncate a string to 'width' printable characters. If 'filterAll'