Merge remote-tracking branch 'upstream/master' into fix-and-ci-static-builds

14 files changed, 244 insertions, 66 deletions

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index ce7cf9754..14399dea3 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -366,11 +366,7 @@ void copyNAR(Source & source, Sink & sink)
 
     ParseSink parseSink; /* null sink; just parse the NAR */
 
-    LambdaSource wrapper([&](unsigned char * data, size_t len) {
-        auto n = source.read(data, len);
-        sink(data, n);
-        return n;
-    });
+    TeeSource wrapper { source, sink };
 
     parseDump(parseSink, wrapper);
 }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 986c5d1cd..147602415 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -3,6 +3,8 @@
 
 #include <glob.h>
 
+#include <nlohmann/json.hpp>
+
 namespace nix {
 
 void Args::addFlag(Flag && flag_)
@@ -205,6 +207,43 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
+nlohmann::json Args::toJSON()
+{
+    auto flags = nlohmann::json::object();
+
+    for (auto & [name, flag] : longFlags) {
+        auto j = nlohmann::json::object();
+        if (flag->shortName)
+            j["shortName"] = std::string(1, flag->shortName);
+        if (flag->description != "")
+            j["description"] = flag->description;
+        if (flag->category != "")
+            j["category"] = flag->category;
+        if (flag->handler.arity != ArityAny)
+            j["arity"] = flag->handler.arity;
+        if (!flag->labels.empty())
+            j["labels"] = flag->labels;
+        flags[name] = std::move(j);
+    }
+
+    auto args = nlohmann::json::array();
+
+    for (auto & arg : expectedArgs) {
+        auto j = nlohmann::json::object();
+        j["label"] = arg.label;
+        j["optional"] = arg.optional;
+        if (arg.handler.arity != ArityAny)
+            j["arity"] = arg.handler.arity;
+        args.push_back(std::move(j));
+    }
+
+    auto res = nlohmann::json::object();
+    res["description"] = description();
+    res["flags"] = std::move(flags);
+    res["args"] = std::move(args);
+    return res;
+}
+
 static void hashTypeCompleter(size_t index, std::string_view prefix) 
 {
     for (auto & type : hashTypes)
@@ -313,11 +352,29 @@ void Command::printHelp(const string & programName, std::ostream & out)
     }
 }
 
+nlohmann::json Command::toJSON()
+{
+    auto exs = nlohmann::json::array();
+
+    for (auto & example : examples()) {
+        auto ex = nlohmann::json::object();
+        ex["description"] = example.description;
+        ex["command"] = chomp(stripIndentation(example.command));
+        exs.push_back(std::move(ex));
+    }
+
+    auto res = Args::toJSON();
+    res["examples"] = std::move(exs);
+    auto s = doc();
+    if (s != "") res.emplace("doc", stripIndentation(s));
+    return res;
+}
+
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
     expectArgs({
-        .label = "command",
+        .label = "subcommand",
         .optional = true,
         .handler = {[=](std::string s) {
             assert(!command);
@@ -387,4 +444,20 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+nlohmann::json MultiCommand::toJSON()
+{
+    auto cmds = nlohmann::json::object();
+
+    for (auto & [name, commandFun] : commands) {
+        auto command = commandFun();
+        auto j = command->toJSON();
+        j["category"] = categories[command->category()];
+        cmds[name] = std::move(j);
+    }
+
+    auto res = Args::toJSON();
+    res["commands"] = std::move(cmds);
+    return res;
+}
+
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 97a517344..3c1f87f7e 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -4,6 +4,8 @@
 #include <map>
 #include <memory>
 
+#include <nlohmann/json_fwd.hpp>
+
 #include "util.hh"
 
 namespace nix {
@@ -20,6 +22,7 @@ public:
 
     virtual void printHelp(const string & programName, std::ostream & out);
 
+    /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
 protected:
@@ -203,6 +206,8 @@ public:
         });
     }
 
+    virtual nlohmann::json toJSON();
+
     friend class MultiCommand;
 };
 
@@ -217,6 +222,9 @@ struct Command : virtual Args
     virtual void prepare() { };
     virtual void run() = 0;
 
+    /* Return documentation about this command, in Markdown format. */
+    virtual std::string doc() { return ""; }
+
     struct Example
     {
         std::string description;
@@ -234,6 +242,8 @@ struct Command : virtual Args
     virtual Category category() { return catDefault; }
 
     void printHelp(const string & programName, std::ostream & out) override;
+
+    nlohmann::json toJSON() override;
 };
 
 typedef std::map<std::string, std::function<ref<Command>()>> Commands;
@@ -259,6 +269,8 @@ public:
     bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
 
     bool processArgs(const Strings & args, bool finish) override;
+
+    nlohmann::json toJSON() override;
 };
 
 Strings argvToStrings(int argc, char * * argv);
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 8fc700a2b..3cf720bce 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -1,6 +1,7 @@
 #include "config.hh"
 #include "args.hh"
-#include "json.hh"
+
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -131,15 +132,18 @@ void Config::resetOverriden()
         s.second.setting->overriden = false;
 }
 
-void Config::toJSON(JSONObject & out)
+nlohmann::json Config::toJSON()
 {
+    auto res = nlohmann::json::object();
     for (auto & s : _settings)
         if (!s.second.isAlias) {
-            JSONObject out2(out.object(s.first));
-            out2.attr("description", s.second.setting->description);
-            JSONPlaceholder out3(out2.placeholder("value"));
-            s.second.setting->toJSON(out3);
+            auto obj = nlohmann::json::object();
+            obj.emplace("description", s.second.setting->description);
+            obj.emplace("aliases", s.second.setting->aliases);
+            obj.emplace("value", s.second.setting->toJSON());
+            res.emplace(s.first, obj);
         }
+    return res;
 }
 
 void Config::convertToArgs(Args & args, const std::string & category)
@@ -153,7 +157,7 @@ AbstractSetting::AbstractSetting(
     const std::string & name,
     const std::string & description,
     const std::set<std::string> & aliases)
-    : name(name), description(description), aliases(aliases)
+    : name(name), description(stripIndentation(description)), aliases(aliases)
 {
 }
 
@@ -162,9 +166,9 @@ void AbstractSetting::setDefault(const std::string & str)
     if (!overriden) set(str);
 }
 
-void AbstractSetting::toJSON(JSONPlaceholder & out)
+nlohmann::json AbstractSetting::toJSON()
 {
-    out.write(to_string());
+    return to_string();
 }
 
 void AbstractSetting::convertToArg(Args & args, const std::string & category)
@@ -172,9 +176,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category)
 }
 
 template<typename T>
-void BaseSetting<T>::toJSON(JSONPlaceholder & out)
+nlohmann::json BaseSetting<T>::toJSON()
 {
-    out.write(value);
+    return value;
 }
 
 template<typename T>
@@ -255,11 +259,9 @@ template<> std::string BaseSetting<Strings>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<Strings>::toJSON(JSONPlaceholder & out)
+template<> nlohmann::json BaseSetting<Strings>::toJSON()
 {
-    JSONList list(out.list());
-    for (auto & s : value)
-        list.elem(s);
+    return value;
 }
 
 template<> void BaseSetting<StringSet>::set(const std::string & str)
@@ -272,11 +274,9 @@ template<> std::string BaseSetting<StringSet>::to_string() const
     return concatStringsSep(" ", value);
 }
 
-template<> void BaseSetting<StringSet>::toJSON(JSONPlaceholder & out)
+template<> nlohmann::json BaseSetting<StringSet>::toJSON()
 {
-    JSONList list(out.list());
-    for (auto & s : value)
-        list.elem(s);
+    return value;
 }
 
 template class BaseSetting<int>;
@@ -323,10 +323,12 @@ void GlobalConfig::resetOverriden()
         config->resetOverriden();
 }
 
-void GlobalConfig::toJSON(JSONObject & out)
+nlohmann::json GlobalConfig::toJSON()
 {
+    auto res = nlohmann::json::object();
     for (auto & config : *configRegistrations)
-        config->toJSON(out);
+        res.update(config->toJSON());
+    return res;
 }
 
 void GlobalConfig::convertToArgs(Args & args, const std::string & category)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 66073546e..2b4265806 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -4,6 +4,8 @@
 
 #include "types.hh"
 
+#include <nlohmann/json_fwd.hpp>
+
 #pragma once
 
 namespace nix {
@@ -42,8 +44,6 @@ namespace nix {
 
 class Args;
 class AbstractSetting;
-class JSONPlaceholder;
-class JSONObject;
 
 class AbstractConfig
 {
@@ -97,7 +97,7 @@ public:
      * Outputs all settings to JSON
      * - out: JSONObject to write the configuration to
      */
-    virtual void toJSON(JSONObject & out) = 0;
+    virtual nlohmann::json toJSON() = 0;
 
     /**
      * Converts settings to `Args` to be used on the command line interface
@@ -167,7 +167,7 @@ public:
 
     void resetOverriden() override;
 
-    void toJSON(JSONObject & out) override;
+    nlohmann::json toJSON() override;
 
     void convertToArgs(Args & args, const std::string & category) override;
 };
@@ -206,7 +206,7 @@ protected:
 
     virtual std::string to_string() const = 0;
 
-    virtual void toJSON(JSONPlaceholder & out);
+    virtual nlohmann::json toJSON();
 
     virtual void convertToArg(Args & args, const std::string & category);
 
@@ -251,7 +251,7 @@ public:
 
     void convertToArg(Args & args, const std::string & category) override;
 
-    void toJSON(JSONPlaceholder & out) override;
+    nlohmann::json toJSON() override;
 };
 
 template<typename T>
@@ -319,7 +319,7 @@ struct GlobalConfig : public AbstractConfig
 
     void resetOverriden() override;
 
-    void toJSON(JSONObject & out) override;
+    nlohmann::json toJSON() override;
 
     void convertToArgs(Args & args, const std::string & category) override;
 
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index dfb3668f1..4a94f0dfd 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -136,6 +136,8 @@ std::string Hash::to_string(Base base, bool includeType) const
     return s;
 }
 
+Hash Hash::dummy(htSHA256);
+
 Hash Hash::parseSRI(std::string_view original) {
     auto rest = original;
 
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 00ce7bb6f..6d6eb70ca 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -59,9 +59,6 @@ private:
     Hash(std::string_view s, HashType type, bool isSRI);
 
 public:
-    /* Check whether a hash is set. */
-    operator bool () const { return (bool) type; }
-
     /* Check whether two hash are equal. */
     bool operator == (const Hash & h2) const;
 
@@ -105,6 +102,8 @@ public:
         assert(type == htSHA1);
         return std::string(to_string(Base16, false), 0, 7);
     }
+
+    static Hash dummy;
 };
 
 /* Helper that defaults empty hashes to the 0 hash. */
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 832aee783..cbbf64395 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -184,6 +184,33 @@ struct JSONLogger : Logger {
         json["action"] = "msg";
         json["level"] = ei.level;
         json["msg"] = oss.str();
+        json["raw_msg"] = ei.hint->str();
+
+        if (ei.errPos.has_value() && (*ei.errPos)) {
+            json["line"] = ei.errPos->line;
+            json["column"] = ei.errPos->column;
+            json["file"] = ei.errPos->file;
+        } else {
+            json["line"] = nullptr;
+            json["column"] = nullptr;
+            json["file"] = nullptr;
+        }
+
+        if (loggerSettings.showTrace.get() && !ei.traces.empty()) {
+            nlohmann::json traces = nlohmann::json::array();
+            for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) {
+                nlohmann::json stackFrame;
+                stackFrame["raw_msg"] = iter->hint.str();
+                if (iter->pos.has_value() && (*iter->pos)) {
+                    stackFrame["line"] = iter->pos->line;
+                    stackFrame["column"] = iter->pos->column;
+                    stackFrame["file"] = iter->pos->file;
+                }
+                traces.push_back(stackFrame);
+            }
+
+            json["trace"] = traces;
+        }
 
         write(json);
     }
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index b75b87f80..77b92fb51 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -37,10 +37,12 @@ typedef uint64_t ActivityId;
 
 struct LoggerSettings : Config
 {
-    Setting<bool> showTrace{this,
-        false,
-        "show-trace",
-        "Whether to show a stack trace on evaluation errors."};
+    Setting<bool> showTrace{
+        this, false, "show-trace",
+        R"(
+          Where Nix should print out a stack trace in case of Nix
+          expression evaluation errors.
+        )"};
 };
 
 extern LoggerSettings loggerSettings;
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index c29c6b29b..a6f1c42e9 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -23,7 +23,8 @@ struct Sink
 };
 
 
-/* A buffered abstract sink. */
+/* A buffered abstract sink. Warning: a BufferedSink should not be
+   used from multiple threads concurrently. */
 struct BufferedSink : virtual Sink
 {
     size_t bufSize, bufPos;
@@ -66,7 +67,8 @@ struct Source
 };
 
 
-/* A buffered abstract source. */
+/* A buffered abstract source. Warning: a BufferedSource should not be
+   used from multiple threads concurrently. */
 struct BufferedSource : Source
 {
     size_t bufSize, bufPosIn, bufPosOut;
@@ -225,6 +227,17 @@ struct SizedSource : Source
     }
 };
 
+/* A sink that that just counts the number of bytes given to it */
+struct LengthSink : Sink
+{
+    uint64_t length = 0;
+
+    virtual void operator () (const unsigned char * _, size_t len)
+    {
+        length += len;
+    }
+};
+
 /* Convert a function into a sink. */
 struct LambdaSink : Sink
 {
diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index 74c59fd31..c5abefe11 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -1,9 +1,9 @@
-#include "json.hh"
 #include "config.hh"
 #include "args.hh"
 
 #include <sstream>
 #include <gtest/gtest.h>
+#include <nlohmann/json.hpp>
 
 namespace nix {
 
@@ -33,7 +33,7 @@ namespace nix {
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "");
-        ASSERT_EQ(iter->second.description, "description");
+        ASSERT_EQ(iter->second.description, "description\n");
     }
 
     TEST(Config, getDefinedOverridenSettingNotSet) {
@@ -59,7 +59,7 @@ namespace nix {
         const auto iter = settings.find("name-of-the-setting");
         ASSERT_NE(iter, settings.end());
         ASSERT_EQ(iter->second.value, "value");
-        ASSERT_EQ(iter->second.description, "description");
+        ASSERT_EQ(iter->second.description, "description\n");
     }
 
     TEST(Config, getDefinedSettingSet2) {
@@ -73,7 +73,7 @@ namespace nix {
         const auto e = settings.find("name-of-the-setting");
         ASSERT_NE(e, settings.end());
         ASSERT_EQ(e->second.value, "value");
-        ASSERT_EQ(e->second.description, "description");
+        ASSERT_EQ(e->second.description, "description\n");
     }
 
     TEST(Config, addSetting) {
@@ -152,29 +152,16 @@ namespace nix {
     }
 
     TEST(Config, toJSONOnEmptyConfig) {
-        std::stringstream out;
-        { // Scoped to force the destructor of JSONObject to write the final `}`
-            JSONObject obj(out);
-            Config config;
-            config.toJSON(obj);
-        }
-
-        ASSERT_EQ(out.str(), "{}");
+        ASSERT_EQ(Config().toJSON().dump(), "{}");
     }
 
     TEST(Config, toJSONOnNonEmptyConfig) {
-        std::stringstream out;
-        { // Scoped to force the destructor of JSONObject to write the final `}`
-            JSONObject obj(out);
-
-            Config config;
-            std::map<std::string, Config::SettingInfo> settings;
-            Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
-            setting.assign("value");
+        Config config;
+        std::map<std::string, Config::SettingInfo> settings;
+        Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
+        setting.assign("value");
 
-            config.toJSON(obj);
-        }
-        ASSERT_EQ(out.str(), R"#({"name-of-the-setting":{"description":"description","value":"value"}})#");
+        ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#");
     }
 
     TEST(Config, setSettingAlias) {
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index ad588055f..7e53f17c6 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -34,6 +34,24 @@ namespace nix {
         }
     }
 
+    TEST(logEI, jsonOutput) {
+        SymbolTable testTable;
+        auto problem_file = testTable.create("random.nix");
+        testing::internal::CaptureStderr();
+
+        makeJSONLogger(*logger)->logEI({
+                .name = "error name",
+                .description = "error without any code lines.",
+                .hint = hintfmt("this hint has %1% templated %2%!!",
+                    "yellow",
+                    "values"),
+                .errPos = Pos(foFile, problem_file, 02, 13)
+            });
+
+        auto str = testing::internal::GetCapturedStderr();
+        ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
+    }
+
     TEST(logEI, appendingHintsToPreviousError) {
 
         MakeError(TestError, Error);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 44b5361dd..16cfa654f 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -1464,6 +1464,47 @@ string base64Decode(std::string_view s)
 }
 
 
+std::string stripIndentation(std::string_view s)
+{
+    size_t minIndent = 10000;
+    size_t curIndent = 0;
+    bool atStartOfLine = true;
+
+    for (auto & c : s) {
+        if (atStartOfLine && c == ' ')
+            curIndent++;
+        else if (c == '\n') {
+            if (atStartOfLine)
+                minIndent = std::max(minIndent, curIndent);
+            curIndent = 0;
+            atStartOfLine = true;
+        } else {
+            if (atStartOfLine) {
+                minIndent = std::min(minIndent, curIndent);
+                atStartOfLine = false;
+            }
+        }
+    }
+
+    std::string res;
+
+    size_t pos = 0;
+    while (pos < s.size()) {
+        auto eol = s.find('\n', pos);
+        if (eol == s.npos) eol = s.size();
+        if (eol - pos > minIndent)
+            res.append(s.substr(pos + minIndent, eol - pos - minIndent));
+        res.push_back('\n');
+        pos = eol + 1;
+    }
+
+    return res;
+}
+
+
+//////////////////////////////////////////////////////////////////////
+
+
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3a20679a8..082e26375 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -464,6 +464,12 @@ string base64Encode(std::string_view s);
 string base64Decode(std::string_view s);
 
 
+/* Remove common leading whitespace from the lines in the string
+   's'. For example, if every line is indented by at least 3 spaces,
+   then we remove 3 spaces from the start of every line. */
+std::string stripIndentation(std::string_view s);
+
+
 /* Get a value for the specified key from an associate container. */
 template <class T>
 std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)