Merge pull request #9099 from obsidiansystems/common-proto

Factor out bits of the worker protocol to use elsewhere

(cherry picked from commit 4b1a97338f517f45e6169d3d8845c5caa5724e97)
Change-Id: If93afa0f8b1cf9b0e705b34fa71e6fd708752758

12 files changed, 508 insertions, 247 deletions

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 83c0a3135..dc4d91079 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -8,8 +8,8 @@
 #include "util.hh"
 #include "archive.hh"
 #include "compression.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "common-protocol.hh"
+#include "common-protocol-impl.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "local-store.hh" // TODO remove, along with remaining downcasts
@@ -1185,11 +1185,11 @@ HookReply DerivationGoal::tryBuildHook()
         throw;
     }
 
-    WorkerProto::WriteConn conn { hook->sink };
+    CommonProto::WriteConn conn { hook->sink };
 
     /* Tell the hook all the inputs that have to be copied to the
        remote system. */
-    WorkerProto::write(worker.store, conn, inputPaths);
+    CommonProto::write(worker.store, conn, inputPaths);
 
     /* Tell the hooks the missing outputs that have to be copied back
        from the remote system. */
@@ -1200,7 +1200,7 @@ HookReply DerivationGoal::tryBuildHook()
             if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
             missingOutputs.insert(outputName);
         }
-        WorkerProto::write(worker.store, conn, missingOutputs);
+        CommonProto::write(worker.store, conn, missingOutputs);
     }
 
     hook->sink = FdSink();
diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh
new file mode 100644
index 000000000..079c182b8
--- /dev/null
+++ b/src/libstore/common-protocol-impl.hh
@@ -0,0 +1,41 @@
+#pragma once
+/**
+ * @file
+ *
+ * Template implementations (as opposed to mere declarations).
+ *
+ * This file is an exmample of the "impl.hh" pattern. See the
+ * contributing guide.
+ */
+
+#include "common-protocol.hh"
+#include "length-prefixed-protocol-helper.hh"
+
+namespace nix {
+
+/* protocol-agnostic templates */
+
+#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
+    TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \
+    { \
+        return LengthPrefixedProtoHelper<CommonProto, T >::read(store, conn); \
+    } \
+    TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \
+    { \
+        LengthPrefixedProtoHelper<CommonProto, T >::write(store, conn, t); \
+    }
+
+COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
+COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::set<T>)
+COMMON_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
+
+#define COMMA_ ,
+COMMON_USE_LENGTH_PREFIX_SERIALISER(
+    template<typename K COMMA_ typename V>,
+    std::map<K COMMA_ V>)
+#undef COMMA_
+
+
+/* protocol-specific templates */
+
+}
diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc
new file mode 100644
index 000000000..f906814bc
--- /dev/null
+++ b/src/libstore/common-protocol.cc
@@ -0,0 +1,98 @@
+#include "serialise.hh"
+#include "util.hh"
+#include "path-with-outputs.hh"
+#include "store-api.hh"
+#include "build-result.hh"
+#include "common-protocol.hh"
+#include "common-protocol-impl.hh"
+#include "archive.hh"
+#include "derivations.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+/* protocol-agnostic definitions */
+
+std::string CommonProto::Serialise<std::string>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    return readString(conn.from);
+}
+
+void CommonProto::Serialise<std::string>::write(const Store & store, CommonProto::WriteConn conn, const std::string & str)
+{
+    conn.to << str;
+}
+
+
+StorePath CommonProto::Serialise<StorePath>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    return store.parseStorePath(readString(conn.from));
+}
+
+void CommonProto::Serialise<StorePath>::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath)
+{
+    conn.to << store.printStorePath(storePath);
+}
+
+
+ContentAddress CommonProto::Serialise<ContentAddress>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    return ContentAddress::parse(readString(conn.from));
+}
+
+void CommonProto::Serialise<ContentAddress>::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca)
+{
+    conn.to << renderContentAddress(ca);
+}
+
+
+Realisation CommonProto::Serialise<Realisation>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    std::string rawInput = readString(conn.from);
+    return Realisation::fromJSON(
+        nlohmann::json::parse(rawInput),
+        "remote-protocol"
+    );
+}
+
+void CommonProto::Serialise<Realisation>::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation)
+{
+    conn.to << realisation.toJSON().dump();
+}
+
+
+DrvOutput CommonProto::Serialise<DrvOutput>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    return DrvOutput::parse(readString(conn.from));
+}
+
+void CommonProto::Serialise<DrvOutput>::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput)
+{
+    conn.to << drvOutput.to_string();
+}
+
+
+std::optional<StorePath> CommonProto::Serialise<std::optional<StorePath>>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    auto s = readString(conn.from);
+    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
+}
+
+void CommonProto::Serialise<std::optional<StorePath>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
+{
+    conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
+}
+
+
+std::optional<ContentAddress> CommonProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, CommonProto::ReadConn conn)
+{
+    return ContentAddress::parseOpt(readString(conn.from));
+}
+
+void CommonProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, CommonProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
+{
+    conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
+}
+
+}
diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh
new file mode 100644
index 000000000..f3f28972a
--- /dev/null
+++ b/src/libstore/common-protocol.hh
@@ -0,0 +1,106 @@
+#pragma once
+///@file
+
+#include "serialise.hh"
+
+namespace nix {
+
+class Store;
+struct Source;
+
+// items being serialized
+class StorePath;
+struct ContentAddress;
+struct DrvOutput;
+struct Realisation;
+
+
+/**
+ * Shared serializers between the worker protocol, serve protocol, and a
+ * few others.
+ *
+ * This `struct` is basically just a `namespace`; We use a type rather
+ * than a namespace just so we can use it as a template argument.
+ */
+struct CommonProto
+{
+    /**
+     * A unidirectional read connection, to be used by the read half of the
+     * canonical serializers below.
+     */
+    struct ReadConn {
+        Source & from;
+    };
+
+    /**
+     * A unidirectional write connection, to be used by the write half of the
+     * canonical serializers below.
+     */
+    struct WriteConn {
+        Sink & to;
+    };
+
+    template<typename T>
+    struct Serialise;
+
+    /**
+     * Wrapper function around `CommonProto::Serialise<T>::write` that allows us to
+     * infer the type instead of having to write it down explicitly.
+     */
+    template<typename T>
+    static void write(const Store & store, WriteConn conn, const T & t)
+    {
+        CommonProto::Serialise<T>::write(store, conn, t);
+    }
+};
+
+#define DECLARE_COMMON_SERIALISER(T) \
+    struct CommonProto::Serialise< T > \
+    { \
+        static T read(const Store & store, CommonProto::ReadConn conn); \
+        static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \
+    }
+
+template<>
+DECLARE_COMMON_SERIALISER(std::string);
+template<>
+DECLARE_COMMON_SERIALISER(StorePath);
+template<>
+DECLARE_COMMON_SERIALISER(ContentAddress);
+template<>
+DECLARE_COMMON_SERIALISER(DrvOutput);
+template<>
+DECLARE_COMMON_SERIALISER(Realisation);
+
+template<typename T>
+DECLARE_COMMON_SERIALISER(std::vector<T>);
+template<typename T>
+DECLARE_COMMON_SERIALISER(std::set<T>);
+template<typename... Ts>
+DECLARE_COMMON_SERIALISER(std::tuple<Ts...>);
+
+#define COMMA_ ,
+template<typename K, typename V>
+DECLARE_COMMON_SERIALISER(std::map<K COMMA_ V>);
+#undef COMMA_
+
+/**
+ * These use the empty string for the null case, relying on the fact
+ * that the underlying types never serialize to the empty string.
+ *
+ * We do this instead of a generic std::optional<T> instance because
+ * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
+ * the same reason, we don't have a std::variant<T..> instances (ordinal
+ * tags 0...n).
+ *
+ * We could the generic instances and then these as specializations for
+ * compatability, but that's proven a bit finnicky, and also makes the
+ * worker protocol harder to implement in other languages where such
+ * specializations may not be allowed.
+ */
+template<>
+DECLARE_COMMON_SERIALISER(std::optional<StorePath>);
+template<>
+DECLARE_COMMON_SERIALISER(std::optional<ContentAddress>);
+
+}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 67069c3c9..fc17e520c 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -4,8 +4,8 @@
 #include "globals.hh"
 #include "util.hh"
 #include "split.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "common-protocol.hh"
+#include "common-protocol-impl.hh"
 #include "fs-accessor.hh"
 #include <boost/container/small_vector.hpp>
 #include <nlohmann/json.hpp>
@@ -895,8 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv,
         drv.outputs.emplace(std::move(name), std::move(output));
     }
 
-    drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store,
-        WorkerProto::ReadConn { .from = in });
+    drv.inputSrcs = CommonProto::Serialise<StorePathSet>::read(store,
+        CommonProto::ReadConn { .from = in });
     in >> drv.platform >> drv.builder;
     drv.args = readStrings<Strings>(in);
 
@@ -944,8 +944,8 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
             },
         }, i.second.raw);
     }
-    WorkerProto::write(store,
-        WorkerProto::WriteConn { .to = out },
+    CommonProto::write(store,
+        CommonProto::WriteConn { .to = out },
         drv.inputSrcs);
     out << drv.platform << drv.builder << drv.args;
     out << drv.env.size();
diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc
index e866aeb42..87b2f8741 100644
--- a/src/libstore/export-import.cc
+++ b/src/libstore/export-import.cc
@@ -1,8 +1,8 @@
 #include "serialise.hh"
 #include "store-api.hh"
 #include "archive.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "common-protocol.hh"
+#include "common-protocol-impl.hh"
 
 #include <algorithm>
 
@@ -46,8 +46,8 @@ void Store::exportPath(const StorePath & path, Sink & sink)
     teeSink
         << exportMagic
         << printStorePath(path);
-    WorkerProto::write(*this,
-        WorkerProto::WriteConn { .to = teeSink },
+    CommonProto::write(*this,
+        CommonProto::WriteConn { .to = teeSink },
         info->references);
     teeSink
         << (info->deriver ? printStorePath(*info->deriver) : "")
@@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
 
         //Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
 
-        auto references = WorkerProto::Serialise<StorePathSet>::read(*this,
-            WorkerProto::ReadConn { .from = source });
+        auto references = CommonProto::Serialise<StorePathSet>::read(*this,
+            CommonProto::ReadConn { .from = source });
         auto deriver = readString(source);
         auto narHash = hashString(htSHA256, saved.s);
 
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 78b05031a..7bf4476d0 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -6,8 +6,8 @@
 #include "build-result.hh"
 #include "store-api.hh"
 #include "path-with-outputs.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "common-protocol.hh"
+#include "common-protocol-impl.hh"
 #include "ssh.hh"
 #include "derivations.hh"
 #include "callback.hh"
@@ -50,37 +50,37 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
         bool good = true;
 
         /**
-         * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the
-         * factored out worker protocol searlizers with a
+         * Coercion to `CommonProto::ReadConn`. This makes it easy to use the
+         * factored out common protocol serialisers with a
          * `LegacySSHStore::Connection`.
          *
-         * The worker protocol connection types are unidirectional, unlike
+         * The common protocol connection types are unidirectional, unlike
          * this type.
          *
-         * @todo Use server protocol serializers, not worker protocol
+         * @todo Use server protocol serializers, not common protocol
          * serializers, once we have made that distiction.
          */
-        operator WorkerProto::ReadConn ()
+        operator CommonProto::ReadConn ()
         {
-            return WorkerProto::ReadConn {
+            return CommonProto::ReadConn {
                 .from = from,
             };
         }
 
         /*
-         * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the
-         * factored out worker protocol searlizers with a
+         * Coercion to `CommonProto::WriteConn`. This makes it easy to use the
+         * factored out common protocol searlizers with a
          * `LegacySSHStore::Connection`.
          *
-         * The worker protocol connection types are unidirectional, unlike
+         * The common protocol connection types are unidirectional, unlike
          * this type.
          *
-         * @todo Use server protocol serializers, not worker protocol
+         * @todo Use server protocol serializers, not common protocol
          * serializers, once we have made that distiction.
          */
-        operator WorkerProto::WriteConn ()
+        operator CommonProto::WriteConn ()
         {
-            return WorkerProto::WriteConn {
+            return CommonProto::WriteConn {
                 .to = to,
             };
         }
@@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             auto deriver = readString(conn->from);
             if (deriver != "")
                 info->deriver = parseStorePath(deriver);
-            info->references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
+            info->references = CommonProto::Serialise<StorePathSet>::read(*this, *conn);
             readLongLong(conn->from); // download size
             info->narSize = readLongLong(conn->from);
 
@@ -217,7 +217,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
                 << printStorePath(info.path)
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << info.narHash.to_string(Base16, false);
-            WorkerProto::write(*this, *conn, info.references);
+            CommonProto::write(*this, *conn, info.references);
             conn->to
                 << info.registrationTime
                 << info.narSize
@@ -246,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
             conn->to
                 << exportMagic
                 << printStorePath(info.path);
-            WorkerProto::write(*this, *conn, info.references);
+            CommonProto::write(*this, *conn, info.references);
             conn->to
                 << (info.deriver ? printStorePath(*info.deriver) : "")
                 << 0
@@ -331,7 +331,7 @@ public:
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
             conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
         if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
-            auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn);
+            auto builtOutputs = CommonProto::Serialise<DrvOutputs>::read(*this, *conn);
             for (auto && [output, realisation] : builtOutputs)
                 status.builtOutputs.insert_or_assign(
                     std::move(output.outputName),
@@ -409,10 +409,10 @@ public:
         conn->to
             << ServeProto::Command::QueryClosure
             << includeOutputs;
-        WorkerProto::write(*this, *conn, paths);
+        CommonProto::write(*this, *conn, paths);
         conn->to.flush();
 
-        for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, *conn))
+        for (auto & i : CommonProto::Serialise<StorePathSet>::read(*this, *conn))
             out.insert(i);
     }
 
@@ -425,10 +425,10 @@ public:
             << ServeProto::Command::QueryValidPaths
             << false // lock
             << maybeSubstitute;
-        WorkerProto::write(*this, *conn, paths);
+        CommonProto::write(*this, *conn, paths);
         conn->to.flush();
 
-        return WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
+        return CommonProto::Serialise<StorePathSet>::read(*this, *conn);
     }
 
     void connect() override
diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh
new file mode 100644
index 000000000..4061b0cd6
--- /dev/null
+++ b/src/libstore/length-prefixed-protocol-helper.hh
@@ -0,0 +1,162 @@
+#pragma once
+/**
+ * @file Reusable serialisers for serialization container types in a
+ * length-prefixed manner.
+ *
+ * Used by both the Worker and Serve protocols.
+ */
+
+#include "types.hh"
+
+namespace nix {
+
+class Store;
+
+/**
+ * Reusable serialisers for serialization container types in a
+ * length-prefixed manner.
+ *
+ * @param T The type of the collection being serialised
+ *
+ * @param Inner This the most important parameter; this is the "inner"
+ * protocol. The user of this will substitute `MyProtocol` or similar
+ * when making a `MyProtocol::Serialiser<Collection<T>>`. Note that the
+ * inside is allowed to call to call `Inner::Serialiser` on different
+ * types. This is especially important for `std::map` which doesn't have
+ * a single `T` but one `K` and one `V`.
+ */
+template<class Inner, typename T>
+struct LengthPrefixedProtoHelper;
+
+/*!
+ * \typedef LengthPrefixedProtoHelper::S
+ *
+ * Read this as simply `using S = Inner::Serialise;`.
+ *
+ * It would be nice to use that directly, but C++ doesn't seem to allow
+ * it. The `typename` keyword needed to refer to `Inner` seems to greedy
+ * (low precedence), and then C++ complains that `Serialise` is not a
+ * type parameter but a real type.
+ *
+ * Making this `S` alias seems to be the only way to avoid these issues.
+ */
+
+#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \
+    struct LengthPrefixedProtoHelper< Inner, T > \
+    { \
+        static T read(const Store & store, typename Inner::ReadConn conn); \
+        static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \
+    private: \
+        template<typename U> using S = typename Inner::template Serialise<U>; \
+    }
+
+template<class Inner, typename T>
+LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector<T>);
+
+template<class Inner, typename T>
+LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set<T>);
+
+template<class Inner, typename... Ts>
+LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple<Ts...>);
+
+template<class Inner, typename K, typename V>
+#define _X std::map<K, V>
+LENGTH_PREFIXED_PROTO_HELPER(Inner, _X);
+#undef _X
+
+template<class Inner, typename T>
+std::vector<T>
+LengthPrefixedProtoHelper<Inner, std::vector<T>>::read(
+    const Store & store, typename Inner::ReadConn conn)
+{
+    std::vector<T> resSet;
+    auto size = readNum<size_t>(conn.from);
+    while (size--) {
+        resSet.push_back(S<T>::read(store, conn));
+    }
+    return resSet;
+}
+
+template<class Inner, typename T>
+void
+LengthPrefixedProtoHelper<Inner, std::vector<T>>::write(
+    const Store & store, typename Inner::WriteConn conn, const std::vector<T> & resSet)
+{
+    conn.to << resSet.size();
+    for (auto & key : resSet) {
+        S<T>::write(store, conn, key);
+    }
+}
+
+template<class Inner, typename T>
+std::set<T>
+LengthPrefixedProtoHelper<Inner, std::set<T>>::read(
+    const Store & store, typename Inner::ReadConn conn)
+{
+    std::set<T> resSet;
+    auto size = readNum<size_t>(conn.from);
+    while (size--) {
+        resSet.insert(S<T>::read(store, conn));
+    }
+    return resSet;
+}
+
+template<class Inner, typename T>
+void
+LengthPrefixedProtoHelper<Inner, std::set<T>>::write(
+    const Store & store, typename Inner::WriteConn conn, const std::set<T> & resSet)
+{
+    conn.to << resSet.size();
+    for (auto & key : resSet) {
+        S<T>::write(store, conn, key);
+    }
+}
+
+template<class Inner, typename K, typename V>
+std::map<K, V>
+LengthPrefixedProtoHelper<Inner, std::map<K, V>>::read(
+    const Store & store, typename Inner::ReadConn conn)
+{
+    std::map<K, V> resMap;
+    auto size = readNum<size_t>(conn.from);
+    while (size--) {
+        auto k = S<K>::read(store, conn);
+        auto v = S<V>::read(store, conn);
+        resMap.insert_or_assign(std::move(k), std::move(v));
+    }
+    return resMap;
+}
+
+template<class Inner, typename K, typename V>
+void
+LengthPrefixedProtoHelper<Inner, std::map<K, V>>::write(
+    const Store & store, typename Inner::WriteConn conn, const std::map<K, V> & resMap)
+{
+    conn.to << resMap.size();
+    for (auto & i : resMap) {
+        S<K>::write(store, conn, i.first);
+        S<V>::write(store, conn, i.second);
+    }
+}
+
+template<class Inner, typename... Ts>
+std::tuple<Ts...>
+LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::read(
+    const Store & store, typename Inner::ReadConn conn)
+{
+    return std::tuple<Ts...> {
+        S<Ts>::read(store, conn)...,
+    };
+}
+
+template<class Inner, typename... Ts>
+void
+LengthPrefixedProtoHelper<Inner, std::tuple<Ts...>>::write(
+    const Store & store, typename Inner::WriteConn conn, const std::tuple<Ts...> & res)
+{
+    std::apply([&]<typename... Us>(const Us &... args) {
+        (S<Us>::write(store, conn, args), ...);
+    }, res);
+}
+
+}
diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh
index 4f797f95a..c043588d6 100644
--- a/src/libstore/worker-protocol-impl.hh
+++ b/src/libstore/worker-protocol-impl.hh
@@ -9,86 +9,51 @@
  */
 
 #include "worker-protocol.hh"
+#include "length-prefixed-protocol-helper.hh"
 
 namespace nix {
 
-template<typename T>
-std::vector<T> WorkerProto::Serialise<std::vector<T>>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    std::vector<T> resSet;
-    auto size = readNum<size_t>(conn.from);
-    while (size--) {
-        resSet.push_back(WorkerProto::Serialise<T>::read(store, conn));
-    }
-    return resSet;
-}
+/* protocol-agnostic templates */
 
-template<typename T>
-void WorkerProto::Serialise<std::vector<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector<T> & resSet)
-{
-    conn.to << resSet.size();
-    for (auto & key : resSet) {
-        WorkerProto::Serialise<T>::write(store, conn, key);
+#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \
+    TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \
+    { \
+        return LengthPrefixedProtoHelper<WorkerProto, T >::read(store, conn); \
+    } \
+    TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \
+    { \
+        LengthPrefixedProtoHelper<WorkerProto, T >::write(store, conn, t); \
     }
-}
 
-template<typename T>
-std::set<T> WorkerProto::Serialise<std::set<T>>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    std::set<T> resSet;
-    auto size = readNum<size_t>(conn.from);
-    while (size--) {
-        resSet.insert(WorkerProto::Serialise<T>::read(store, conn));
-    }
-    return resSet;
-}
+WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::vector<T>)
+WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename T>, std::set<T>)
+WORKER_USE_LENGTH_PREFIX_SERIALISER(template<typename... Ts>, std::tuple<Ts...>)
 
-template<typename T>
-void WorkerProto::Serialise<std::set<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::set<T> & resSet)
-{
-    conn.to << resSet.size();
-    for (auto & key : resSet) {
-        WorkerProto::Serialise<T>::write(store, conn, key);
-    }
-}
+#define COMMA_ ,
+WORKER_USE_LENGTH_PREFIX_SERIALISER(
+    template<typename K COMMA_ typename V>,
+    std::map<K COMMA_ V>)
+#undef COMMA_
 
-template<typename K, typename V>
-std::map<K, V> WorkerProto::Serialise<std::map<K, V>>::read(const Store & store, WorkerProto::ReadConn conn)
+/**
+ * Use `CommonProto` where possible.
+ */
+template<typename T>
+struct WorkerProto::Serialise
 {
-    std::map<K, V> resMap;
-    auto size = readNum<size_t>(conn.from);
-    while (size--) {
-        auto k = WorkerProto::Serialise<K>::read(store, conn);
-        auto v = WorkerProto::Serialise<V>::read(store, conn);
-        resMap.insert_or_assign(std::move(k), std::move(v));
+    static T read(const Store & store, WorkerProto::ReadConn conn)
+    {
+        return CommonProto::Serialise<T>::read(store,
+            CommonProto::ReadConn { .from = conn.from });
     }
-    return resMap;
-}
-
-template<typename K, typename V>
-void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, WorkerProto::WriteConn conn, const std::map<K, V> & resMap)
-{
-    conn.to << resMap.size();
-    for (auto & i : resMap) {
-        WorkerProto::Serialise<K>::write(store, conn, i.first);
-        WorkerProto::Serialise<V>::write(store, conn, i.second);
+    static void write(const Store & store, WorkerProto::WriteConn conn, const T & t)
+    {
+        CommonProto::Serialise<T>::write(store,
+            CommonProto::WriteConn { .to = conn.to },
+            t);
     }
-}
-
-template<typename... Ts>
-std::tuple<Ts...> WorkerProto::Serialise<std::tuple<Ts...>>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    return std::tuple<Ts...> {
-        WorkerProto::Serialise<Ts>::read(store, conn)...,
-    };
-}
+};
 
-template<typename... Ts>
-void WorkerProto::Serialise<std::tuple<Ts...>>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple<Ts...> & res)
-{
-    std::apply([&]<typename... Us>(const Us &... args) {
-        (WorkerProto::Serialise<Us>::write(store, conn, args), ...);
-    }, res);
-}
+/* protocol-specific templates */
 
 }
diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc
index a23130743..415e66f16 100644
--- a/src/libstore/worker-protocol.cc
+++ b/src/libstore/worker-protocol.cc
@@ -12,27 +12,7 @@
 
 namespace nix {
 
-std::string WorkerProto::Serialise<std::string>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    return readString(conn.from);
-}
-
-void WorkerProto::Serialise<std::string>::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str)
-{
-    conn.to << str;
-}
-
-
-StorePath WorkerProto::Serialise<StorePath>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    return store.parseStorePath(readString(conn.from));
-}
-
-void WorkerProto::Serialise<StorePath>::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath)
-{
-    conn.to << store.printStorePath(storePath);
-}
-
+/* protocol-specific definitions */
 
 std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn)
 {
@@ -68,17 +48,6 @@ void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & sto
 }
 
 
-ContentAddress WorkerProto::Serialise<ContentAddress>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    return ContentAddress::parse(readString(conn.from));
-}
-
-void WorkerProto::Serialise<ContentAddress>::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca)
-{
-    conn.to << renderContentAddress(ca);
-}
-
-
 DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     auto s = readString(conn.from);
@@ -91,32 +60,6 @@ void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto
 }
 
 
-Realisation WorkerProto::Serialise<Realisation>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    std::string rawInput = readString(conn.from);
-    return Realisation::fromJSON(
-        nlohmann::json::parse(rawInput),
-        "remote-protocol"
-    );
-}
-
-void WorkerProto::Serialise<Realisation>::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation)
-{
-    conn.to << realisation.toJSON().dump();
-}
-
-
-DrvOutput WorkerProto::Serialise<DrvOutput>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    return DrvOutput::parse(readString(conn.from));
-}
-
-void WorkerProto::Serialise<DrvOutput>::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput)
-{
-    conn.to << drvOutput.to_string();
-}
-
-
 KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn)
 {
     auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn);
@@ -168,26 +111,4 @@ void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto
 }
 
 
-std::optional<StorePath> WorkerProto::Serialise<std::optional<StorePath>>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    auto s = readString(conn.from);
-    return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s);
-}
-
-void WorkerProto::Serialise<std::optional<StorePath>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
-{
-    conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
-}
-
-
-std::optional<ContentAddress> WorkerProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, WorkerProto::ReadConn conn)
-{
-    return ContentAddress::parseOpt(readString(conn.from));
-}
-
-void WorkerProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<ContentAddress> & caOpt)
-{
-    conn.to << (caOpt ? renderContentAddress(*caOpt) : "");
-}
-
 }
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b7f42f24d..c84060103 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -1,7 +1,7 @@
 #pragma once
 ///@file
 
-#include "serialise.hh"
+#include "common-protocol.hh"
 
 namespace nix {
 
@@ -28,11 +28,7 @@ class Store;
 struct Source;
 
 // items being serialised
-class StorePath;
-struct ContentAddress;
 struct DerivedPath;
-struct DrvOutput;
-struct Realisation;
 struct BuildResult;
 struct KeyedBuildResult;
 enum TrustedFlag : bool;
@@ -193,60 +189,32 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op)
  * be legal specialization syntax. See below for what that looks like in
  * practice.
  */
-#define MAKE_WORKER_PROTO(T) \
-    struct WorkerProto::Serialise< T > { \
+#define DECLARE_WORKER_SERIALISER(T) \
+    struct WorkerProto::Serialise< T > \
+    { \
         static T read(const Store & store, WorkerProto::ReadConn conn); \
         static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \
     };
 
 template<>
-MAKE_WORKER_PROTO(std::string);
+DECLARE_WORKER_SERIALISER(DerivedPath);
 template<>
-MAKE_WORKER_PROTO(StorePath);
+DECLARE_WORKER_SERIALISER(BuildResult);
 template<>
-MAKE_WORKER_PROTO(ContentAddress);
+DECLARE_WORKER_SERIALISER(KeyedBuildResult);
 template<>
-MAKE_WORKER_PROTO(DerivedPath);
-template<>
-MAKE_WORKER_PROTO(DrvOutput);
-template<>
-MAKE_WORKER_PROTO(Realisation);
-template<>
-MAKE_WORKER_PROTO(BuildResult);
-template<>
-MAKE_WORKER_PROTO(KeyedBuildResult);
-template<>
-MAKE_WORKER_PROTO(std::optional<TrustedFlag>);
+DECLARE_WORKER_SERIALISER(std::optional<TrustedFlag>);
 
 template<typename T>
-MAKE_WORKER_PROTO(std::vector<T>);
+DECLARE_WORKER_SERIALISER(std::vector<T>);
 template<typename T>
-MAKE_WORKER_PROTO(std::set<T>);
+DECLARE_WORKER_SERIALISER(std::set<T>);
 template<typename... Ts>
-MAKE_WORKER_PROTO(std::tuple<Ts...>);
+DECLARE_WORKER_SERIALISER(std::tuple<Ts...>);
 
+#define COMMA_ ,
 template<typename K, typename V>
-#define X_ std::map<K, V>
-MAKE_WORKER_PROTO(X_);
-#undef X_
-
-/**
- * These use the empty string for the null case, relying on the fact
- * that the underlying types never serialise to the empty string.
- *
- * We do this instead of a generic std::optional<T> instance because
- * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For
- * the same reason, we don't have a std::variant<T..> instances (ordinal
- * tags 0...n).
- *
- * We could the generic instances and then these as specializations for
- * compatability, but that's proven a bit finnicky, and also makes the
- * worker protocol harder to implement in other languages where such
- * specializations may not be allowed.
- */
-template<>
-MAKE_WORKER_PROTO(std::optional<StorePath>);
-template<>
-MAKE_WORKER_PROTO(std::optional<ContentAddress>);
+DECLARE_WORKER_SERIALISER(std::map<K COMMA_ V>);
+#undef COMMA_
 
 }
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 96c3f7d7e..6fc22214a 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -11,8 +11,8 @@
 #include "serve-protocol.hh"
 #include "shared.hh"
 #include "util.hh"
-#include "worker-protocol.hh"
-#include "worker-protocol-impl.hh"
+#include "common-protocol.hh"
+#include "common-protocol-impl.hh"
 #include "graphml.hh"
 #include "legacy.hh"
 #include "path-with-outputs.hh"
@@ -821,8 +821,8 @@ static void opServe(Strings opFlags, Strings opArgs)
     out.flush();
     unsigned int clientVersion = readInt(in);
 
-    WorkerProto::ReadConn rconn { .from = in };
-    WorkerProto::WriteConn wconn { .to = out };
+    CommonProto::ReadConn rconn { .from = in };
+    CommonProto::WriteConn wconn { .to = out };
 
     auto getBuildSettings = [&]() {
         // FIXME: changing options here doesn't work if we're
@@ -867,7 +867,7 @@ static void opServe(Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryValidPaths: {
                 bool lock = readInt(in);
                 bool substitute = readInt(in);
-                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
                 if (lock && writeAllowed)
                     for (auto & path : paths)
                         store->addTempRoot(path);
@@ -876,19 +876,19 @@ static void opServe(Strings opFlags, Strings opArgs)
                     store->substitutePaths(paths);
                 }
 
-                WorkerProto::write(*store, wconn, store->queryValidPaths(paths));
+                CommonProto::write(*store, wconn, store->queryValidPaths(paths));
                 break;
             }
 
             case ServeProto::Command::QueryPathInfos: {
-                auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
+                auto paths = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
                 // !!! Maybe we want a queryPathInfos?
                 for (auto & i : paths) {
                     try {
                         auto info = store->queryPathInfo(i);
                         out << store->printStorePath(info->path)
                             << (info->deriver ? store->printStorePath(*info->deriver) : "");
-                        WorkerProto::write(*store, wconn, info->references);
+                        CommonProto::write(*store, wconn, info->references);
                         // !!! Maybe we want compression?
                         out << info->narSize // downloadSize
                             << info->narSize;
@@ -916,7 +916,7 @@ static void opServe(Strings opFlags, Strings opArgs)
 
             case ServeProto::Command::ExportPaths: {
                 readInt(in); // obsolete
-                store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, rconn), out);
+                store->exportPaths(CommonProto::Serialise<StorePathSet>::read(*store, rconn), out);
                 break;
             }
 
@@ -962,7 +962,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                     DrvOutputs builtOutputs;
                     for (auto & [output, realisation] : status.builtOutputs)
                         builtOutputs.insert_or_assign(realisation.id, realisation);
-                    WorkerProto::write(*store, wconn, builtOutputs);
+                    CommonProto::write(*store, wconn, builtOutputs);
                 }
 
                 break;
@@ -971,9 +971,9 @@ static void opServe(Strings opFlags, Strings opArgs)
             case ServeProto::Command::QueryClosure: {
                 bool includeOutputs = readInt(in);
                 StorePathSet closure;
-                store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, rconn),
+                store->computeFSClosure(CommonProto::Serialise<StorePathSet>::read(*store, rconn),
                     closure, false, includeOutputs);
-                WorkerProto::write(*store, wconn, closure);
+                CommonProto::write(*store, wconn, closure);
                 break;
             }
 
@@ -988,7 +988,7 @@ static void opServe(Strings opFlags, Strings opArgs)
                 };
                 if (deriver != "")
                     info.deriver = store->parseStorePath(deriver);
-                info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn);
+                info.references = CommonProto::Serialise<StorePathSet>::read(*store, rconn);
                 in >> info.registrationTime >> info.narSize >> info.ultimate;
                 info.sigs = readStrings<StringSet>(in);
                 info.ca = ContentAddress::parseOpt(readString(in));