Merge pull request #7887 from obsidiansystems/add-derivation

`nix derivation add`, `show-derivation` -> `derivation show`

6 files changed, 266 insertions, 75 deletions

diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh
index 19fdfc1eb..368a7ec48 100644
--- a/src/libstore/content-address.hh
+++ b/src/libstore/content-address.hh
@@ -3,6 +3,7 @@
 
 #include <variant>
 #include "hash.hh"
+#include "comparator.hh"
 
 namespace nix {
 
@@ -30,6 +31,8 @@ struct TextHash {
      * Hash of the contents of the text/file.
      */
     Hash hash;
+
+    GENERATE_CMP(TextHash, me->hash);
 };
 
 /**
@@ -46,6 +49,8 @@ struct FixedOutputHash {
     Hash hash;
 
     std::string printMethodAlgo() const;
+
+    GENERATE_CMP(FixedOutputHash, me->method, me->hash);
 };
 
 /**
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 06cc69056..abdfb1978 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -889,6 +889,67 @@ std::optional<BasicDerivation> Derivation::tryResolve(
     return resolved;
 }
 
+
+void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const
+{
+    assert(drvPath.isDerivation());
+    std::string drvName(drvPath.name());
+    drvName = drvName.substr(0, drvName.size() - drvExtension.size());
+
+    if (drvName != name) {
+        throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name);
+    }
+
+    auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
+    {
+        auto j = env.find(varName);
+        if (j == env.end() || store.parseStorePath(j->second) != actual)
+            throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
+                store.printStorePath(drvPath), varName, store.printStorePath(actual));
+    };
+
+
+    // Don't need the answer, but do this anyways to assert is proper
+    // combination. The code below is more general and naturally allows
+    // combinations that are currently prohibited.
+    type();
+
+    std::optional<DrvHash> hashesModulo;
+    for (auto & i : outputs) {
+        std::visit(overloaded {
+            [&](const DerivationOutput::InputAddressed & doia) {
+                if (!hashesModulo) {
+                    // somewhat expensive so we do lazily
+                    hashesModulo = hashDerivationModulo(store, *this, true);
+                }
+                auto currentOutputHash = get(hashesModulo->hashes, i.first);
+                if (!currentOutputHash)
+                    throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
+                        store.printStorePath(drvPath), store.printStorePath(doia.path), i.first);
+                StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName);
+                if (doia.path != recomputed)
+                    throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
+                        store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed));
+                envHasRightPath(doia.path, i.first);
+            },
+            [&](const DerivationOutput::CAFixed & dof) {
+                StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
+                envHasRightPath(path, i.first);
+            },
+            [&](const DerivationOutput::CAFloating &) {
+                /* Nothing to check */
+            },
+            [&](const DerivationOutput::Deferred &) {
+                /* Nothing to check */
+            },
+            [&](const DerivationOutput::Impure &) {
+                /* Nothing to check */
+            },
+        }, i.second.raw());
+    }
+}
+
+
 const Hash impureOutputHash = hashString(htSHA256, "impure");
 
 nlohmann::json DerivationOutput::toJSON(
@@ -916,10 +977,79 @@ nlohmann::json DerivationOutput::toJSON(
     return res;
 }
 
+
+DerivationOutput DerivationOutput::fromJSON(
+    const Store & store, std::string_view drvName, std::string_view outputName,
+    const nlohmann::json & _json)
+{
+    std::set<std::string_view> keys;
+    auto json = (std::map<std::string, nlohmann::json>) _json;
+
+    for (const auto & [key, _] : json)
+        keys.insert(key);
+
+    auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> {
+        std::string hashAlgo = json["hashAlgo"];
+        auto method = FileIngestionMethod::Flat;
+        if (hashAlgo.substr(0, 2) == "r:") {
+            method = FileIngestionMethod::Recursive;
+            hashAlgo = hashAlgo.substr(2);
+        }
+        auto hashType = parseHashType(hashAlgo);
+        return { method, hashType };
+    };
+
+    if (keys == (std::set<std::string_view> { "path" })) {
+        return DerivationOutput::InputAddressed {
+            .path = store.parseStorePath((std::string) json["path"]),
+        };
+    }
+
+    else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
+        auto [method, hashType] = methodAlgo();
+        auto dof = DerivationOutput::CAFixed {
+            .hash = {
+                .method = method,
+                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
+            },
+        };
+        if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
+            throw Error("Path doesn't match derivation output");
+        return dof;
+    }
+
+    else if (keys == (std::set<std::string_view> { "hashAlgo" })) {
+        auto [method, hashType] = methodAlgo();
+        return DerivationOutput::CAFloating {
+            .method = method,
+            .hashType = hashType,
+        };
+    }
+
+    else if (keys == (std::set<std::string_view> { })) {
+        return DerivationOutput::Deferred {};
+    }
+
+    else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) {
+        auto [method, hashType] = methodAlgo();
+        return DerivationOutput::Impure {
+            .method = method,
+            .hashType = hashType,
+        };
+    }
+
+    else {
+        throw Error("invalid JSON for derivation output");
+    }
+}
+
+
 nlohmann::json Derivation::toJSON(const Store & store) const
 {
     nlohmann::json res = nlohmann::json::object();
 
+    res["name"] = name;
+
     {
         nlohmann::json & outputsObj = res["outputs"];
         outputsObj = nlohmann::json::object();
@@ -950,4 +1080,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const
     return res;
 }
 
+
+Derivation Derivation::fromJSON(
+    const Store & store,
+    const nlohmann::json & json)
+{
+    Derivation res;
+
+    res.name = json["name"];
+
+    {
+        auto & outputsObj = json["outputs"];
+        for (auto & [outputName, output] : outputsObj.items()) {
+            res.outputs.insert_or_assign(
+                outputName,
+                DerivationOutput::fromJSON(store, res.name, outputName, output));
+        }
+    }
+
+    {
+        auto & inputsList = json["inputSrcs"];
+        for (auto & input : inputsList)
+            res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
+    }
+
+    {
+        auto & inputDrvsObj = json["inputDrvs"];
+        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
+            res.inputDrvs[store.parseStorePath(inputDrvPath)] =
+                static_cast<const StringSet &>(inputOutputs);
+    }
+
+    res.platform = json["system"];
+    res.builder = json["builder"];
+    res.args = json["args"];
+    res.env = json["env"];
+
+    return res;
+}
+
 }
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index e12bd2119..9e7ceeb5d 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -7,6 +7,7 @@
 #include "content-address.hh"
 #include "repair-flag.hh"
 #include "sync.hh"
+#include "comparator.hh"
 
 #include <map>
 #include <variant>
@@ -24,6 +25,8 @@ class Store;
 struct DerivationOutputInputAddressed
 {
     StorePath path;
+
+    GENERATE_CMP(DerivationOutputInputAddressed, me->path);
 };
 
 /**
@@ -44,6 +47,8 @@ struct DerivationOutputCAFixed
      * @param outputName The name of this output.
      */
     StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+
+    GENERATE_CMP(DerivationOutputCAFixed, me->hash);
 };
 
 /**
@@ -62,13 +67,17 @@ struct DerivationOutputCAFloating
      * How the serialization will be hashed
      */
     HashType hashType;
+
+    GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType);
 };
 
 /**
  * Input-addressed output which depends on a (CA) derivation whose hash
  * isn't known yet.
  */
-struct DerivationOutputDeferred {};
+struct DerivationOutputDeferred {
+    GENERATE_CMP(DerivationOutputDeferred);
+};
 
 /**
  * Impure output which is moved to a content-addressed location (like
@@ -85,6 +94,8 @@ struct DerivationOutputImpure
      * How the serialization will be hashed
      */
     HashType hashType;
+
+    GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType);
 };
 
 typedef std::variant<
@@ -125,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw
         const Store & store,
         std::string_view drvName,
         std::string_view outputName) const;
+    static DerivationOutput fromJSON(
+        const Store & store,
+        std::string_view drvName,
+        std::string_view outputName,
+        const nlohmann::json & json);
 };
 
 typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@@ -273,6 +289,15 @@ struct BasicDerivation
     DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const;
 
     static std::string_view nameFromPath(const StorePath & storePath);
+
+    GENERATE_CMP(BasicDerivation,
+        me->outputs,
+        me->inputSrcs,
+        me->platform,
+        me->builder,
+        me->args,
+        me->env,
+        me->name);
 };
 
 struct Derivation : BasicDerivation
@@ -308,11 +333,26 @@ struct Derivation : BasicDerivation
         Store & store,
         const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
 
+    /* Check that the derivation is valid and does not present any
+       illegal states.
+
+       This is mainly a matter of checking the outputs, where our C++
+       representation supports all sorts of combinations we do not yet
+       allow. */
+    void checkInvariants(Store & store, const StorePath & drvPath) const;
+
     Derivation() = default;
     Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
     Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
 
     nlohmann::json toJSON(const Store & store) const;
+    static Derivation fromJSON(
+        const Store & store,
+        const nlohmann::json & json);
+
+    GENERATE_CMP(Derivation,
+        static_cast<const BasicDerivation &>(*me),
+        me->inputDrvs);
 };
 
 
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index e0ad50f6d..c3be1b461 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -710,62 +710,6 @@ void canonicalisePathMetaData(const Path & path,
     canonicalisePathMetaData(path, uidRange, inodesSeen);
 }
 
-
-void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv)
-{
-    assert(drvPath.isDerivation());
-    std::string drvName(drvPath.name());
-    drvName = drvName.substr(0, drvName.size() - drvExtension.size());
-
-    auto envHasRightPath = [&](const StorePath & actual, const std::string & varName)
-    {
-        auto j = drv.env.find(varName);
-        if (j == drv.env.end() || parseStorePath(j->second) != actual)
-            throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'",
-                printStorePath(drvPath), varName, printStorePath(actual));
-    };
-
-
-    // Don't need the answer, but do this anyways to assert is proper
-    // combination. The code below is more general and naturally allows
-    // combinations that are currently prohibited.
-    drv.type();
-
-    std::optional<DrvHash> hashesModulo;
-    for (auto & i : drv.outputs) {
-        std::visit(overloaded {
-            [&](const DerivationOutput::InputAddressed & doia) {
-                if (!hashesModulo) {
-                    // somewhat expensive so we do lazily
-                    hashesModulo = hashDerivationModulo(*this, drv, true);
-                }
-                auto currentOutputHash = get(hashesModulo->hashes, i.first);
-                if (!currentOutputHash)
-                    throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'",
-                        printStorePath(drvPath), printStorePath(doia.path), i.first);
-                StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName);
-                if (doia.path != recomputed)
-                    throw Error("derivation '%s' has incorrect output '%s', should be '%s'",
-                        printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
-                envHasRightPath(doia.path, i.first);
-            },
-            [&](const DerivationOutput::CAFixed & dof) {
-                StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
-                envHasRightPath(path, i.first);
-            },
-            [&](const DerivationOutput::CAFloating &) {
-                /* Nothing to check */
-            },
-            [&](const DerivationOutput::Deferred &) {
-                /* Nothing to check */
-            },
-            [&](const DerivationOutput::Impure &) {
-                /* Nothing to check */
-            },
-        }, i.second.raw());
-    }
-}
-
 void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
 {
     experimentalFeatureSettings.require(Xp::CaDerivations);
@@ -876,7 +820,7 @@ uint64_t LocalStore::addValidPath(State & state,
            derivations).  Note that if this throws an error, then the
            DB transaction is rolled back, so the path validity
            registration above is undone. */
-        if (checkOutputs) checkDerivationOutputs(info.path, drv);
+        if (checkOutputs) drv.checkInvariants(*this, info.path);
 
         for (auto & i : drv.outputsAndOptPaths(*this)) {
             /* Floating CA derivations have indeterminate output paths until
@@ -1175,8 +1119,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
         for (auto & [_, i] : infos)
             if (i.path.isDerivation()) {
                 // FIXME: inefficient; we already loaded the derivation in addValidPath().
-                checkDerivationOutputs(i.path,
-                    readInvalidDerivation(i.path));
+                readInvalidDerivation(i.path).checkInvariants(*this, i.path);
             }
 
         /* Do a topological sort of the paths.  This will throw an
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 6deaa051f..7e0849961 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -270,8 +270,6 @@ private:
 
     std::pair<Path, AutoCloseFD> createTempDirInStore();
 
-    void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
-
     typedef std::unordered_set<ino_t> InodeHash;
 
     InodeHash loadInodeHash();
diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc
index 12be8504d..80ee52fd0 100644
--- a/src/libstore/tests/derivation.cc
+++ b/src/libstore/tests/derivation.cc
@@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest
 {
 };
 
-#define TEST_JSON(TYPE, NAME, STR, VAL, ...)                           \
-    TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) {            \
-        using nlohmann::literals::operator "" _json;                   \
-        ASSERT_EQ(                                                     \
-            STR ## _json,                                              \
-            (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__));  \
+#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
+    TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) {    \
+        using nlohmann::literals::operator "" _json;           \
+        ASSERT_EQ(                                             \
+            STR ## _json,                                      \
+            (DerivationOutput { VAL }).toJSON(                 \
+                *store,                                        \
+                DRV_NAME,                                      \
+                OUTPUT_NAME));                                 \
+    }                                                          \
+                                                               \
+    TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) {  \
+        using nlohmann::literals::operator "" _json;           \
+        ASSERT_EQ(                                             \
+            DerivationOutput { VAL },                          \
+            DerivationOutput::fromJSON(                        \
+                *store,                                        \
+                DRV_NAME,                                      \
+                OUTPUT_NAME,                                   \
+                STR ## _json));                                \
     }
 
-TEST_JSON(DerivationOutput, inputAddressed,
+TEST_JSON(inputAddressed,
     R"({
         "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
     })",
@@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(DerivationOutput, caFixed,
+TEST_JSON(caFixed,
     R"({
         "hashAlgo": "r:sha256",
         "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
@@ -42,7 +56,7 @@ TEST_JSON(DerivationOutput, caFixed,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(DerivationOutput, caFloating,
+TEST_JSON(caFloating,
     R"({
         "hashAlgo": "r:sha256"
     })",
@@ -52,12 +66,12 @@ TEST_JSON(DerivationOutput, caFloating,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(DerivationOutput, deferred,
+TEST_JSON(deferred,
     R"({ })",
     DerivationOutput::Deferred { },
     "drv-name", "output-name")
 
-TEST_JSON(DerivationOutput, impure,
+TEST_JSON(impure,
     R"({
         "hashAlgo": "r:sha256",
         "impure": true
@@ -68,8 +82,28 @@ TEST_JSON(DerivationOutput, impure,
     }),
     "drv-name", "output-name")
 
-TEST_JSON(Derivation, impure,
+#undef TEST_JSON
+
+#define TEST_JSON(NAME, STR, VAL, DRV_NAME)                     \
+    TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) {   \
+        using nlohmann::literals::operator "" _json;            \
+        ASSERT_EQ(                                              \
+            STR ## _json,                                       \
+            (Derivation { VAL }).toJSON(*store));               \
+    }                                                           \
+                                                                \
+    TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \
+        using nlohmann::literals::operator "" _json;            \
+        ASSERT_EQ(                                              \
+            Derivation { VAL },                                 \
+            Derivation::fromJSON(                               \
+                *store,                                         \
+                STR ## _json));                                 \
+    }
+
+TEST_JSON(simple,
     R"({
+      "name": "my-derivation",
       "inputSrcs": [
         "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
       ],
@@ -92,6 +126,7 @@ TEST_JSON(Derivation, impure,
     })",
     ({
         Derivation drv;
+        drv.name = "my-derivation";
         drv.inputSrcs = {
             store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"),
         };
@@ -117,7 +152,8 @@ TEST_JSON(Derivation, impure,
             },
         };
         drv;
-    }))
+    }),
+    "drv-name")
 
 #undef TEST_JSON