libutil: begin porting serialization to generators

generators are a better basis for serializers than streaming into sinks
as we do currently for many reasons, such as being usable as sources if
one wishes to (without requiring an intermediate sink to serialize full
data sets into memory, or boost coroutines to turn sinks into sources),
composing more naturally (as one can just yield a sub-generator instead
of being forced to wrap entire substreams into clunky functions or even
more clunky custom types to implement operator<< on), allowing wrappers
to transform data with clear ownership semantics (removing the need for
explicit memory allocations and Source wrappers), and many other things

Change-Id: I361d89ff556354f6930d9204f55117565f2f7f20

2 files changed, 88 insertions, 54 deletions

diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 80b111f08..11bc183cc 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -330,55 +330,40 @@ void writePadding(size_t len, Sink & sink)
 }
 
 
-void writeString(std::string_view data, Sink & sink)
+WireFormatGenerator SerializingTransform::operator()(std::string_view s)
 {
-    sink << data.size();
-    sink(data);
-    writePadding(data.size(), sink);
+    co_yield s.size();
+    co_yield Bytes(s.begin(), s.size());
+    co_yield SerializingTransform::padding(s.size());
 }
 
-
-Sink & operator << (Sink & sink, std::string_view s)
-{
-    writeString(s, sink);
-    return sink;
-}
-
-
-template<class T> void writeStrings(const T & ss, Sink & sink)
-{
-    sink << ss.size();
-    for (auto & i : ss)
-        sink << i;
-}
-
-Sink & operator << (Sink & sink, const Strings & s)
+WireFormatGenerator SerializingTransform::operator()(const Strings & ss)
 {
-    writeStrings(s, sink);
-    return sink;
+    co_yield ss.size();
+    for (const auto & s : ss)
+        co_yield std::string_view(s);
 }
 
-Sink & operator << (Sink & sink, const StringSet & s)
+WireFormatGenerator SerializingTransform::operator()(const StringSet & ss)
 {
-    writeStrings(s, sink);
-    return sink;
+    co_yield ss.size();
+    for (const auto & s : ss)
+        co_yield std::string_view(s);
 }
 
-Sink & operator << (Sink & sink, const Error & ex)
+WireFormatGenerator SerializingTransform::operator()(const Error & ex)
 {
     auto & info = ex.info();
-    sink
-        << "Error"
-        << info.level
-        << "Error" // removed
-        << info.msg.str()
-        << 0 // FIXME: info.errPos
-        << info.traces.size();
+    co_yield "Error";
+    co_yield info.level;
+    co_yield "Error"; // removed
+    co_yield info.msg.str();
+    co_yield 0; // FIXME: info.errPos
+    co_yield info.traces.size();
     for (auto & trace : info.traces) {
-        sink << 0; // FIXME: trace.pos
-        sink << trace.hint.str();
+        co_yield 0; // FIXME: trace.pos
+        co_yield trace.hint.str();
     }
-    return sink;
 }
 
 
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 491b1987d..2651ec979 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -350,33 +350,82 @@ inline Sink & operator<<(Sink & sink, Generator<Bytes> && g)
     return sink;
 }
 
+struct SerializingTransform;
+using WireFormatGenerator = Generator<Bytes, SerializingTransform>;
+
+struct SerializingTransform
+{
+    std::array<unsigned char, 8> buf;
+
+    Bytes operator()(uint64_t n)
+    {
+        buf[0] = n & 0xff;
+        buf[1] = (n >> 8) & 0xff;
+        buf[2] = (n >> 16) & 0xff;
+        buf[3] = (n >> 24) & 0xff;
+        buf[4] = (n >> 32) & 0xff;
+        buf[5] = (n >> 40) & 0xff;
+        buf[6] = (n >> 48) & 0xff;
+        buf[7] = (unsigned char) (n >> 56) & 0xff;
+        return {reinterpret_cast<const char *>(buf.begin()), 8};
+    }
+
+    static Bytes padding(size_t unpadded)
+    {
+        return Bytes("\0\0\0\0\0\0\0", unpadded % 8 ? 8 - unpadded % 8 : 0);
+    }
+
+    // opt in to generator chaining. without this co_yielding
+    // another generator of any type will cause a type error.
+    auto operator()(Generator<Bytes> && g)
+    {
+        return std::move(g);
+    }
+
+    // only choose this for *exactly* char spans, do not allow implicit
+    // conversions. this would cause ambiguities with strings literals,
+    // and resolving those with more string-like overloads needs a lot.
+    template<typename Span>
+        requires std::same_as<Span, std::span<char>> || std::same_as<Span, std::span<const char>>
+    Bytes operator()(Span s)
+    {
+        return s;
+    }
+    WireFormatGenerator operator()(std::string_view s);
+    WireFormatGenerator operator()(const Strings & s);
+    WireFormatGenerator operator()(const StringSet & s);
+    WireFormatGenerator operator()(const Error & s);
+};
+
 void writePadding(size_t len, Sink & sink);
-void writeString(std::string_view s, Sink & sink);
 
-inline Sink & operator << (Sink & sink, uint64_t n)
+inline Sink & operator<<(Sink & sink, uint64_t u)
 {
-    unsigned char buf[8];
-    buf[0] = n & 0xff;
-    buf[1] = (n >> 8) & 0xff;
-    buf[2] = (n >> 16) & 0xff;
-    buf[3] = (n >> 24) & 0xff;
-    buf[4] = (n >> 32) & 0xff;
-    buf[5] = (n >> 40) & 0xff;
-    buf[6] = (n >> 48) & 0xff;
-    buf[7] = (unsigned char) (n >> 56) & 0xff;
-    sink({(char *) buf, sizeof(buf)});
-    return sink;
+    return sink << [&]() -> WireFormatGenerator { co_yield u; }();
+}
+
+inline Sink & operator<<(Sink & sink, std::string_view s)
+{
+    return sink << [&]() -> WireFormatGenerator { co_yield s; }();
+}
+
+inline Sink & operator<<(Sink & sink, const Strings & s)
+{
+    return sink << [&]() -> WireFormatGenerator { co_yield s; }();
 }
 
-Sink & operator << (Sink & in, const Error & ex);
-Sink & operator << (Sink & sink, std::string_view s);
-Sink & operator << (Sink & sink, const Strings & s);
-Sink & operator << (Sink & sink, const StringSet & s);
+inline Sink & operator<<(Sink & sink, const StringSet & s)
+{
+    return sink << [&]() -> WireFormatGenerator { co_yield s; }();
+}
 
+inline Sink & operator<<(Sink & sink, const Error & ex)
+{
+    return sink << [&]() -> WireFormatGenerator { co_yield ex; }();
+}
 
 MakeError(SerialisationError, Error);
 
-
 template<typename T>
 T readNum(Source & source)
 {