From fe9cbe838c3d59e2768b92d8cab0e5a2674f5bfb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Feb 2023 18:37:35 -0500 Subject: Create `Derivation::fromJSON` And test, of course --- src/libstore/content-address.hh | 5 ++ src/libstore/derivations.cc | 105 +++++++++++++++++++++++++++++++++++++++ src/libstore/derivations.hh | 35 ++++++++++++- src/libstore/tests/derivation.cc | 61 ++++++++++++++++++----- src/libutil/comparator.hh | 10 ++-- 5 files changed, 197 insertions(+), 19 deletions(-) diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 19fdfc1eb..368a7ec48 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -3,6 +3,7 @@ #include #include "hash.hh" +#include "comparator.hh" namespace nix { @@ -30,6 +31,8 @@ struct TextHash { * Hash of the contents of the text/file. */ Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /** @@ -46,6 +49,8 @@ struct FixedOutputHash { Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /** diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 06cc69056..ea4abb352 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -889,6 +889,7 @@ std::optional Derivation::tryResolve( return resolved; } + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( @@ -916,6 +917,73 @@ nlohmann::json DerivationOutput::toJSON( return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json) +{ + std::set keys; + auto json = (std::map) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgo = json["hashAlgo"]; + auto method = FileIngestionMethod::Flat; + if (hashAlgo.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = hashAlgo.substr(2); + } + auto hashType = parseHashType(hashAlgo); + return { method, hashType }; + }; + + if (keys == (std::set { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .hash = { + .method = method, + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set { "hashAlgo" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = method, + .hashType = hashType, + }; + } + + else if (keys == (std::set { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set { "hashAlgo", "impure" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = method, + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); @@ -950,4 +1018,41 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, std::string_view drvName, + const nlohmann::json & json) +{ + Derivation res; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, drvName, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index e12bd2119..0722dc61d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -7,6 +7,7 @@ #include "content-address.hh" #include "repair-flag.hh" #include "sync.hh" +#include "comparator.hh" #include #include @@ -24,6 +25,8 @@ class Store; struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; /** @@ -44,6 +47,8 @@ struct DerivationOutputCAFixed * @param outputName The name of this output. */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->hash); }; /** @@ -62,13 +67,17 @@ struct DerivationOutputCAFloating * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; /** * Input-addressed output which depends on a (CA) derivation whose hash * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; /** * Impure output which is moved to a content-addressed location (like @@ -85,6 +94,8 @@ struct DerivationOutputImpure * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -125,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json); }; typedef std::map DerivationOutputs; @@ -273,6 +289,15 @@ struct BasicDerivation DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation @@ -313,6 +338,14 @@ struct Derivation : BasicDerivation Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + std::string_view drvName, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast(*me), + me->inputDrvs); }; diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 12be8504d..8993cd80b 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest { }; -#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ - TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ +#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json)); \ } -TEST_JSON(DerivationOutput, inputAddressed, +TEST_JSON(inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixed, +TEST_JSON(caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -42,7 +56,7 @@ TEST_JSON(DerivationOutput, caFixed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFloating, +TEST_JSON(caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -52,12 +66,12 @@ TEST_JSON(DerivationOutput, caFloating, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, deferred, +TEST_JSON(deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(DerivationOutput, impure, +TEST_JSON(impure, R"({ "hashAlgo": "r:sha256", "impure": true @@ -68,7 +82,27 @@ TEST_JSON(DerivationOutput, impure, }), "drv-name", "output-name") -TEST_JSON(Derivation, impure, +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + DRV_NAME, \ + STR ## _json)); \ + } + +TEST_JSON(simple, R"({ "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" @@ -117,7 +151,8 @@ TEST_JSON(Derivation, impure, }, }; drv; - })) + }), + "drv-name") #undef TEST_JSON diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 2b5424b3d..9f661c5c3 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -17,12 +17,12 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ + __VA_OPT__(const MY_TYPE* me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -- cgit v1.2.3 From b200784cec056da53378eb043cae4fad188e4e6f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 30 Mar 2023 11:06:52 -0400 Subject: Include the name in the JSON for derivations This is non-breaking change in the to-JSON direction. This *is* a breaking change in the from-JSON direction, but we don't care, as that is brand new in this PR. `nix show-derivation --help` currently has the sole public documentation of this format, it is updated accordingly. --- src/libstore/derivations.cc | 8 ++++++-- src/libstore/derivations.hh | 1 - src/libstore/tests/derivation.cc | 3 ++- src/nix/show-derivation.md | 3 +++ 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index ea4abb352..9f74f1c30 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -988,6 +988,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -1020,17 +1022,19 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( - const Store & store, std::string_view drvName, + const Store & store, const nlohmann::json & json) { Derivation res; + res.name = json["name"]; + { auto & outputsObj = json["outputs"]; for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, - DerivationOutput::fromJSON(store, drvName, outputName, output)); + DerivationOutput::fromJSON(store, res.name, outputName, output)); } } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 0722dc61d..38727d77a 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -340,7 +340,6 @@ struct Derivation : BasicDerivation nlohmann::json toJSON(const Store & store) const; static Derivation fromJSON( const Store & store, - std::string_view drvName, const nlohmann::json & json); GENERATE_CMP(Derivation, diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 8993cd80b..80ee52fd0 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -98,12 +98,12 @@ TEST_JSON(impure, Derivation { VAL }, \ Derivation::fromJSON( \ *store, \ - DRV_NAME, \ STR ## _json)); \ } TEST_JSON(simple, R"({ + "name": "my-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -126,6 +126,7 @@ TEST_JSON(simple, })", ({ Derivation drv; + drv.name = "my-derivation"; drv.inputSrcs = { store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md index 1d37c6f5a..17d88a5f9 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/show-derivation.md @@ -51,6 +51,9 @@ The JSON output is a JSON object whose keys are the store paths of the derivations, and whose values are a JSON object with the following fields: +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these -- cgit v1.2.3 From 4e9f32f993aaa9f7995919e480e0e920d946184d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Mar 2023 16:57:36 -0500 Subject: Liberate `checkDerivationOutputs` from `LocalStore` Make it instead a method on `Derivation` that can work with any store. We will need this for a CLI command to create a derivation. --- src/libstore/derivations.cc | 60 ++++++++++++++++++++++++++++++++++++++++++++ src/libstore/derivations.hh | 8 ++++++ src/libstore/local-store.cc | 61 ++------------------------------------------- src/libstore/local-store.hh | 2 -- 4 files changed, 70 insertions(+), 61 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9f74f1c30..abdfb1978 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -890,6 +890,66 @@ std::optional Derivation::tryResolve( } +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + StorePath path = store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 38727d77a..9e7ceeb5d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -333,6 +333,14 @@ struct Derivation : BasicDerivation Store & store, const std::map, StorePath> & inputDrvOutputs) const; + /* Check that the derivation is valid and does not present any + illegal states. + + This is mainly a matter of checking the outputs, where our C++ + representation supports all sorts of combinations we do not yet + allow. */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e0ad50f6d..c3be1b461 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -710,62 +710,6 @@ void canonicalisePathMetaData(const Path & path, canonicalisePathMetaData(path, uidRange, inodesSeen); } - -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -876,7 +820,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -1175,8 +1119,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6deaa051f..7e0849961 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -270,8 +270,6 @@ private: std::pair createTempDirInStore(); - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); - typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); -- cgit v1.2.3 From 2b98af2e62f8a70cfa132f9bac7049a198309df8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 13:59:19 -0400 Subject: `nix show-derivation` -> `nix derivation show` --- doc/manual/src/glossary.md | 2 +- src/nix/derivation.cc | 25 +++++++++++++++++++++++++ src/nix/main.cc | 1 + src/nix/show-derivation.cc | 4 ++-- src/nix/show-derivation.md | 6 +++--- tests/ca/build.sh | 2 +- tests/impure-derivations.sh | 4 ++-- 7 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 src/nix/derivation.cc diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index fe5920746..4eedb2e93 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -15,7 +15,7 @@ Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - See [`nix show-derivation`](./command-ref/new-cli/nix3-show-derivation.md) (experimental) for displaying the contents of store derivations. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. [store derivation]: #gloss-store-derivation diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc new file mode 100644 index 000000000..cd3975a4f --- /dev/null +++ b/src/nix/derivation.cc @@ -0,0 +1,25 @@ +#include "command.hh" + +using namespace nix; + +struct CmdDerivation : virtual NixMultiCommand +{ + CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + { } + + std::string description() override + { + return "Work with derivations, Nix's notion of a build plan."; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix derivation' requires a sub-command."); + command->second->run(); + } +}; + +static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 4d4164333..f943f77bb 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -127,6 +127,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, + {"show-derivation", {"derivation", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 4a406ae08..0a55b36f2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -1,5 +1,5 @@ // FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? #include "command.hh" #include "common-args.hh" @@ -61,4 +61,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto rCmdShowDerivation = registerCommand("show-derivation"); +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md index 17d88a5f9..1ebe22f65 100644 --- a/src/nix/show-derivation.md +++ b/src/nix/show-derivation.md @@ -8,7 +8,7 @@ R""( [store derivation]: ../../glossary.md#gloss-store-derivation ```console - # nix show-derivation nixpkgs#hello + # nix derivation show nixpkgs#hello { "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { … @@ -20,14 +20,14 @@ R""( NixOS system: ```console - # nix show-derivation -r /run/current-system + # nix derivation show -r /run/current-system ``` * Print all files fetched using `fetchurl` by Firefox's dependency graph: ```console - # nix show-derivation -r nixpkgs#firefox \ + # nix derivation show -r nixpkgs#firefox \ | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ | uniq | sort ``` diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 98e1c5125..7754ad276 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -3,7 +3,7 @@ source common.sh drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) -nix show-derivation "$drv" --arg seed 1 +nix derivation show "$drv" --arg seed 1 buildAttr () { local derivationPath=$1 diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh index 7595fdd35..c7dadf397 100644 --- a/tests/impure-derivations.sh +++ b/tests/impure-derivations.sh @@ -37,8 +37,8 @@ path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnIm (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) -- cgit v1.2.3 From 27597f813126bf0e866c6f6ba35dc7d3761843f8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Apr 2023 14:05:53 -0400 Subject: Rename files to reflect new `nix derivation show` name This will match the files we added for `nix add derivation` in the rest of this PR. --- src/nix/derivation-show.cc | 64 +++++++++++++++++++++++++++ src/nix/derivation-show.md | 108 +++++++++++++++++++++++++++++++++++++++++++++ src/nix/show-derivation.cc | 64 --------------------------- src/nix/show-derivation.md | 108 --------------------------------------------- 4 files changed, 172 insertions(+), 172 deletions(-) create mode 100644 src/nix/derivation-show.cc create mode 100644 src/nix/derivation-show.md delete mode 100644 src/nix/show-derivation.cc delete mode 100644 src/nix/show-derivation.md diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc new file mode 100644 index 000000000..bf637246d --- /dev/null +++ b/src/nix/derivation-show.cc @@ -0,0 +1,64 @@ +// FIXME: integrate this with nix path-info? +// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdShowDerivation : InstallablesCommand +{ + bool recursive = false; + + CmdShowDerivation() + { + addFlag({ + .longName = "recursive", + .shortName = 'r', + .description = "Include the dependencies of the specified derivations.", + .handler = {&recursive, true} + }); + } + + std::string description() override + { + return "show the contents of a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-show.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store, Installables && installables) override + { + auto drvPaths = Installable::toDerivations(store, installables, true); + + if (recursive) { + StorePathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = std::move(closure); + } + + json jsonRoot = json::object(); + + for (auto & drvPath : drvPaths) { + if (!drvPath.isDerivation()) continue; + + jsonRoot[store->printStorePath(drvPath)] = + store->readDerivation(drvPath).toJSON(*store); + } + logger->cout(jsonRoot.dump(2)); + } +}; + +static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md new file mode 100644 index 000000000..1ebe22f65 --- /dev/null +++ b/src/nix/derivation-show.md @@ -0,0 +1,108 @@ +R""( + +# Examples + +* Show the [store derivation] that results from evaluating the Hello + package: + + [store derivation]: ../../glossary.md#gloss-store-derivation + + ```console + # nix derivation show nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix derivation show -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix derivation show -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations +are used internally by Nix. They are store paths with extension `.drv` +that represent the build-time dependency graph to which a Nix +expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +The JSON output is a JSON object whose keys are the store paths of the +derivations, and whose values are a JSON object with the following +fields: + +* `name`: The name of the derivation. This is used when calculating the + store paths of the derivation's outputs. + +* `outputs`: Information about the output paths of the + derivation. This is a JSON object with one member per output, where + the key is the output name and the value is a JSON object with these + fields: + + * `path`: The output path. + * `hashAlgo`: For fixed-output derivations, the hashing algorithm + (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a + NAR hash rather than a flat file hash. + * `hash`: For fixed-output derivations, the expected content hash in + base-16. + + Example: + + ```json + "outputs": { + "out": { + "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + "hashAlgo": "r:sha256", + "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + } + } + ``` + +* `inputSrcs`: A list of store paths on which this derivation depends. + +* `inputDrvs`: A JSON object specifying the derivations on which this + derivation depends, and what outputs of those derivations. For + example, + + ```json + "inputDrvs": { + "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + } + ``` + + specifies that this derivation depends on the `dev` output of + `curl`, and the `out` output of `unzip`. + +* `system`: The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: The absolute path of the program to be executed to run + the build. Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: The command-line arguments passed to the `builder`. + +* `env`: The environment passed to the `builder`. + +)"" diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc deleted file mode 100644 index 0a55b36f2..000000000 --- a/src/nix/show-derivation.cc +++ /dev/null @@ -1,64 +0,0 @@ -// FIXME: integrate this with nix path-info? -// FIXME: rename to 'nix store derivation show' or 'nix debug derivation show'? - -#include "command.hh" -#include "common-args.hh" -#include "store-api.hh" -#include "archive.hh" -#include "derivations.hh" -#include - -using namespace nix; -using json = nlohmann::json; - -struct CmdShowDerivation : InstallablesCommand -{ - bool recursive = false; - - CmdShowDerivation() - { - addFlag({ - .longName = "recursive", - .shortName = 'r', - .description = "Include the dependencies of the specified derivations.", - .handler = {&recursive, true} - }); - } - - std::string description() override - { - return "show the contents of a store derivation"; - } - - std::string doc() override - { - return - #include "show-derivation.md" - ; - } - - Category category() override { return catUtility; } - - void run(ref store, Installables && installables) override - { - auto drvPaths = Installable::toDerivations(store, installables, true); - - if (recursive) { - StorePathSet closure; - store->computeFSClosure(drvPaths, closure); - drvPaths = std::move(closure); - } - - json jsonRoot = json::object(); - - for (auto & drvPath : drvPaths) { - if (!drvPath.isDerivation()) continue; - - jsonRoot[store->printStorePath(drvPath)] = - store->readDerivation(drvPath).toJSON(*store); - } - logger->cout(jsonRoot.dump(2)); - } -}; - -static auto rCmdShowDerivation = registerCommand2({"derivation", "show"}); diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md deleted file mode 100644 index 1ebe22f65..000000000 --- a/src/nix/show-derivation.md +++ /dev/null @@ -1,108 +0,0 @@ -R""( - -# Examples - -* Show the [store derivation] that results from evaluating the Hello - package: - - [store derivation]: ../../glossary.md#gloss-store-derivation - - ```console - # nix derivation show nixpkgs#hello - { - "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { - … - } - } - ``` - -* Show the full derivation graph (if available) that produced your - NixOS system: - - ```console - # nix derivation show -r /run/current-system - ``` - -* Print all files fetched using `fetchurl` by Firefox's dependency - graph: - - ```console - # nix derivation show -r nixpkgs#firefox \ - | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ - | uniq | sort - ``` - - Note that `.outputs.out.hash` selects *fixed-output derivations* - (derivations that produce output with a specified content hash), - while `.env.urls` selects derivations with a `urls` attribute. - -# Description - -This command prints on standard output a JSON representation of the -[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. - -By default, this command only shows top-level derivations, but with -`--recursive`, it also shows their dependencies. - -The JSON output is a JSON object whose keys are the store paths of the -derivations, and whose values are a JSON object with the following -fields: - -* `name`: The name of the derivation. This is used when calculating the - store paths of the derivation's outputs. - -* `outputs`: Information about the output paths of the - derivation. This is a JSON object with one member per output, where - the key is the output name and the value is a JSON object with these - fields: - - * `path`: The output path. - * `hashAlgo`: For fixed-output derivations, the hashing algorithm - (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a - NAR hash rather than a flat file hash. - * `hash`: For fixed-output derivations, the expected content hash in - base-16. - - Example: - - ```json - "outputs": { - "out": { - "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - "hashAlgo": "r:sha256", - "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" - } - } - ``` - -* `inputSrcs`: A list of store paths on which this derivation depends. - -* `inputDrvs`: A JSON object specifying the derivations on which this - derivation depends, and what outputs of those derivations. For - example, - - ```json - "inputDrvs": { - "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] - } - ``` - - specifies that this derivation depends on the `dev` output of - `curl`, and the `out` output of `unzip`. - -* `system`: The system type on which this derivation is to be built - (e.g. `x86_64-linux`). - -* `builder`: The absolute path of the program to be executed to run - the build. Typically this is the `bash` shell - (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). - -* `args`: The command-line arguments passed to the `builder`. - -* `env`: The environment passed to the `builder`. - -)"" -- cgit v1.2.3 From 59e072871410ba88694cf2597079a9f0f115e458 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Feb 2023 18:37:35 -0500 Subject: Create `nix derivation add` command Also refine `nix derivation show`'s docs very slightly. --- src/nix/derivation-add.cc | 45 +++++++++++++++++++++++++++++++++++++++++++++ src/nix/derivation-add.md | 18 ++++++++++++++++++ src/nix/derivation-show.md | 9 +++++---- tests/ca/derivation-json.sh | 26 ++++++++++++++++++++++++++ tests/derivation-json.sh | 12 ++++++++++++ tests/local.mk | 2 ++ 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 src/nix/derivation-add.cc create mode 100644 src/nix/derivation-add.md create mode 100644 tests/ca/derivation-json.sh create mode 100644 tests/derivation-json.sh diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc new file mode 100644 index 000000000..4d91d4538 --- /dev/null +++ b/src/nix/derivation-add.cc @@ -0,0 +1,45 @@ +// FIXME: rename to 'nix plan add' or 'nix derivation add'? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "derivations.hh" +#include + +using namespace nix; +using json = nlohmann::json; + +struct CmdAddDerivation : MixDryRun, StoreCommand +{ + std::string description() override + { + return "Add a store derivation"; + } + + std::string doc() override + { + return + #include "derivation-add.md" + ; + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); + + auto drv = Derivation::fromJSON(*store, json); + + auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); + + drv.checkInvariants(*store, drvPath); + + writeDerivation(*store, drv, NoRepair, dryRun); + + logger->cout("%s", store->printStorePath(drvPath)); + } +}; + +static auto rCmdAddDerivation = registerCommand2({"derivation", "add"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md new file mode 100644 index 000000000..f116681ab --- /dev/null +++ b/src/nix/derivation-add.md @@ -0,0 +1,18 @@ +R""( + +# Description + +This command reads from standard input a JSON representation of a +[store derivation] to which an [*installable*](./nix.md#installables) evaluates. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. + +[store derivation]: ../../glossary.md#gloss-store-derivation + +The JSON format is documented under the [`derivation show`] command. + +[`derivation show`]: ./nix3-derivation-show.md + +)"" diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 1ebe22f65..1296e2885 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -39,10 +39,11 @@ R""( # Description This command prints on standard output a JSON representation of the -[store derivation]s to which [*installables*](./nix.md#installables) evaluate. Store derivations -are used internally by Nix. They are store paths with extension `.drv` -that represent the build-time dependency graph to which a Nix -expression evaluates. +[store derivation]s to which [*installables*](./nix.md#installables) evaluate. + +Store derivations are used internally by Nix. They are store paths with +extension `.drv` that represent the build-time dependency graph to which +a Nix expression evaluates. By default, this command only shows top-level derivations, but with `--recursive`, it also shows their dependencies. diff --git a/tests/ca/derivation-json.sh b/tests/ca/derivation-json.sh new file mode 100644 index 000000000..3615177e9 --- /dev/null +++ b/tests/ca/derivation-json.sh @@ -0,0 +1,26 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +drvPath=$(nix-instantiate ../simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Content-addressed derivations can be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json +drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +# With --dry-run nothing is actually written +[[ ! -e "$drvPath3" ]] + +# Without --dry-run it is actually written +drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +[[ "$drvPath4" = "$drvPath3" ]] +[[ -e "$drvPath3" ]] + +# The modified derivation read back as JSON matches +nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json +diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json diff --git a/tests/derivation-json.sh b/tests/derivation-json.sh new file mode 100644 index 000000000..b6be5d977 --- /dev/null +++ b/tests/derivation-json.sh @@ -0,0 +1,12 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json + +drvPath2=$(nix derivation add < $TEST_HOME/simple.json) + +[[ "$drvPath" = "$drvPath2" ]] + +# Input addressed derivations cannot be renamed. +jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/local.mk b/tests/local.mk index 7d54d94e5..6cb466e8e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -102,6 +102,8 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ ca/why-depends.sh \ + derivation-json.sh \ + ca/derivation-json.sh \ import-derivation.sh \ ca/import-derivation.sh \ nix_path.sh \ -- cgit v1.2.3 From 9d1105824f99ada218387eb8791643f66b25c3f8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 3 Apr 2023 09:18:33 -0400 Subject: Add release notes for `nix derivation {add,show}` --- doc/manual/src/release-notes/rl-next.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index af1850bdc..5b62836bf 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -45,3 +45,14 @@ This is useful for scripting interactions with (non-legacy-ssh) remote Nix stores. `nix store ping` and `nix doctor` now display this information. + +* A new command `nix derivation add` is created, to allow adding derivations to the store without involving the Nix language. + It exists to round out our collection of basic utility/plumbing commands, and allow for a low barrier-to-entry way of experimenting with alternative front-ends to the Nix Store. + It uses the same JSON layout as `nix show-derivation`, and is its inverse. + +* `nix show-derivation` has been renamed to `nix derivation show`. + This matches `nix derivation add`, and avoids bloating the top-level namespace. + The old name is still kept as an alias for compatibility, however. + +* The `nix derivation {add,show}` JSON format now includes the derivation name as a top-level field. + This is useful in general, but especially necessary for the `add` direction, as otherwise we would need to pass in the name out of band for certain cases. -- cgit v1.2.3