From 6a5a8f51bb9773ab4aeac8f508872bcf5ec6fda2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 17 May 2023 14:59:47 +0200 Subject: add cross-references to pure evaluation mode use consistent wording everywhere. add some details on the configuration option documentation. --- src/libutil/experimental-features.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad0ec0427..9bebf77e5 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -50,6 +50,8 @@ constexpr std::array xpFeatureDetails = {{ or other impure derivations can rely on impure derivations. Finally, an impure derivation cannot also be [content-addressed](#xp-feature-ca-derivations). + + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). )", }, { -- cgit v1.2.3 From b9e5ce4a27f4a8bbee1a2eeb6fddbf569cbfdd7a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 11 May 2023 18:01:41 -0400 Subject: Upgrade `downstreamPlaceholder` to a type with methods This gets us ready for dynamic derivation dependencies (part of RFC 92). --- src/libutil/experimental-features.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad0ec0427..5aae0347b 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -207,6 +207,9 @@ constexpr std::array xpFeatureDetails = {{ - "text hashing" derivation outputs, so we can build .drv files. + + - dependencies in derivations on the outputs of + derivations that are themselves derivations outputs. )", }, }}; -- cgit v1.2.3 From 4c4ae887b871ace7bc8528bc3dcf4fe85a92ff09 Mon Sep 17 00:00:00 2001 From: Konstantin Vukolov Date: Thu, 18 May 2023 13:18:34 +0300 Subject: Add option isInteractive --- src/libutil/util.cc | 14 ++++++++++++-- src/libutil/util.hh | 3 ++- 2 files changed, 14 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 21d1c8dcd..3a8309149 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1141,9 +1141,9 @@ std::vector stringsToCharPtrs(const Strings & ss) } std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input) + const std::optional & input, bool isInteractive) { - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); if (!statusOk(res.first)) throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); @@ -1193,6 +1193,16 @@ void runProgram2(const RunOptions & options) // case), so we can't use it if we alter the environment processOptions.allowVfork = !options.environment; + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + /* Fork. */ Pid pid = startProcess([&]() { if (options.environment) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 040fed68f..a7907cd14 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -415,7 +415,7 @@ pid_t startProcess(std::function fun, const ProcessOptions & options = P */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::optional & input = {}); + const std::optional & input = {}, bool isInteractive = false); struct RunOptions { @@ -430,6 +430,7 @@ struct RunOptions Source * standardIn = nullptr; Sink * standardOut = nullptr; bool mergeStderrToStdout = false; + bool isInteractive = false; }; std::pair runProgram(RunOptions && options); -- cgit v1.2.3 From 3ebe1341abe1b0ad59bd4925517af18d9200f818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: Make `RewritingSink` accept a map of rewrites Giving it the same semantics as `rewriteStrings`. Also add some tests for it --- src/libutil/references.cc | 140 ++++++++++++++++++++++++++++++++++++++++ src/libutil/references.hh | 56 ++++++++++++++++ src/libutil/tests/references.cc | 46 +++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 src/libutil/references.cc create mode 100644 src/libutil/references.hh create mode 100644 src/libutil/tests/references.cc (limited to 'src/libutil') diff --git a/src/libutil/references.cc b/src/libutil/references.cc new file mode 100644 index 000000000..74003584a --- /dev/null +++ b/src/libutil/references.cc @@ -0,0 +1,140 @@ +#include "references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include +#include +#include +#include + + +namespace nix { + + +static size_t refLength = 32; /* characters */ + + +static void search( + std::string_view s, + StringSet & hashes, + StringSet & seen) +{ + static std::once_flag initialised; + static bool isBase32[256]; + std::call_once(initialised, [](){ + for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; + for (unsigned int i = 0; i < base32Chars.size(); ++i) + isBase32[(unsigned char) base32Chars[i]] = true; + }); + + for (size_t i = 0; i + refLength <= s.size(); ) { + int j; + bool match = true; + for (j = refLength - 1; j >= 0; --j) + if (!isBase32[(unsigned char) s[i + j]]) { + i += j + 1; + match = false; + break; + } + if (!match) continue; + std::string ref(s.substr(i, refLength)); + if (hashes.erase(ref)) { + debug("found reference to '%1%' at offset '%2%'", ref, i); + seen.insert(ref); + } + ++i; + } +} + + +void RefScanSink::operator () (std::string_view data) +{ + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + auto s = tail; + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); + search(s, hashes, seen); + + search(data, hashes, seen); + + auto rest = refLength - tailLen; + if (rest < tail.size()) + tail = tail.substr(tail.size() - rest); + tail.append(data.data() + data.size() - tailLen, tailLen); +} + + +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : RewritingSink({{from, to}}, nextSink) +{ +} + +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) +{ + long unsigned int maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); + } + this->maxRewriteSize = maxRewriteSize; +} + +void RewritingSink::operator () (std::string_view data) +{ + std::string s(prev); + s.append(data); + + s = rewriteStrings(s, rewrites); + + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); + + auto consumed = s.size() - prev.size(); + + pos += consumed; + + if (consumed) nextSink(s.substr(0, consumed)); +} + +void RewritingSink::flush() +{ + if (prev.empty()) return; + pos += prev.size(); + nextSink(prev); + prev.clear(); +} + +HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) + : hashSink(ht) + , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) +{ +} + +void HashModuloSink::operator () (std::string_view data) +{ + rewritingSink(data); +} + +HashResult HashModuloSink::finish() +{ + rewritingSink.flush(); + + /* Hash the positions of the self-references. This ensures that a + NAR with self-references and a NAR with some of the + self-references already zeroed out do not produce a hash + collision. FIXME: proof. */ + for (auto & pos : rewritingSink.matches) + hashSink(fmt("|%d", pos)); + + auto h = hashSink.finish(); + return {h.first, rewritingSink.pos}; +} + +} diff --git a/src/libutil/references.hh b/src/libutil/references.hh new file mode 100644 index 000000000..ffd730e7b --- /dev/null +++ b/src/libutil/references.hh @@ -0,0 +1,56 @@ +#pragma once +///@file + +#include "hash.hh" + +namespace nix { + +class RefScanSink : public Sink +{ + StringSet hashes; + StringSet seen; + + std::string tail; + +public: + + RefScanSink(StringSet && hashes) : hashes(hashes) + { } + + StringSet & getResult() + { return seen; } + + void operator () (std::string_view data) override; +}; + +struct RewritingSink : Sink +{ + const StringMap rewrites; + long unsigned int maxRewriteSize; + std::string prev; + Sink & nextSink; + uint64_t pos = 0; + + std::vector matches; + + RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); + + void operator () (std::string_view data) override; + + void flush(); +}; + +struct HashModuloSink : AbstractHashSink +{ + HashSink hashSink; + RewritingSink rewritingSink; + + HashModuloSink(HashType ht, const std::string & modulus); + + void operator () (std::string_view data) override; + + HashResult finish() override; +}; + +} diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc new file mode 100644 index 000000000..a517d9aa1 --- /dev/null +++ b/src/libutil/tests/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + -- cgit v1.2.3 From be4890747051de0e489d608fdba65489c45d2b02 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 17 Feb 2023 19:23:09 +0100 Subject: ci: Always run with sandbox, even on Darwin And fix a test failure in the sandbox due to /home existing on Darwin but not being accessible in the sandbox since it's a symlink to /System/Volumes/Data/home, see https://github.com/NixOS/nix/actions/runs/4205378453/jobs/7297384658#step:6:2127: C++ exception with description "error: getting status of /home/schnitzel/darmstadt/pommes: Operation not permitted" thrown in the test body. On Linux this wasn't a problem because there /home doesn't exist in the sandbox --- src/libutil/tests/tests.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 250e83a38..f3c1e8248 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -202,7 +202,7 @@ namespace nix { } TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } /* ---------------------------------------------------------------------------- -- cgit v1.2.3 From 2c462486fe0c1bcb5b1142507d2875e98b2418df Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 26 May 2023 15:32:28 +0200 Subject: create pathAccessible, use it to infer default dirs --- src/libutil/util.cc | 11 +++++++++++ src/libutil/util.hh | 8 ++++++++ 2 files changed, 19 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3a8309149..aa0a154fd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -266,6 +266,17 @@ bool pathExists(const Path & path) return false; } +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + Path readLink(const Path & path) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index a7907cd14..00fcb9b79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -120,6 +120,14 @@ struct stat lstat(const Path & path); */ bool pathExists(const Path & path); +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + /** * Read the contents (target) of a symbolic link. The result is not * in any way canonicalised. -- cgit v1.2.3 From 3c78920f7358957dba37a379e9d0b062dd3192e2 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 9 Jun 2023 17:53:18 +0800 Subject: Parse TOML timestamps (#8120) Currently `fromTOML` throws an exception when encountering a timestamp since the Nix language lacks a way to represent them. This patch changes this beaviour and makes `fromTOML` parse timestamps as attrsets of the format { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; } This is guarded by an experimental feature flag to leave room for iterating on the representation. --- src/libutil/experimental-features.cc | 11 +++++++++-- src/libutil/experimental-features.hh | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 4bf545d7b..c4642d333 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -214,6 +214,13 @@ constexpr std::array xpFeatureDetails = {{ derivations that are themselves derivations outputs. )", }, + { + .tag = Xp::ParseTomlTimestamps, + .name = "parse-toml-timestamps", + .description = R"( + Allow parsing of timestamps in builtins.fromTOML. + )", + }, }}; static_assert( @@ -248,7 +255,7 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -nlohmann::json documentExperimentalFeatures() +nlohmann::json documentExperimentalFeatures() { StringMap res; for (auto & xpFeature : xpFeatureDetails) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 409100592..892c6c371 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -30,6 +30,7 @@ enum struct ExperimentalFeature DiscardReferences, DaemonTrustOverride, DynamicDerivations, + ParseTomlTimestamps, }; /** -- cgit v1.2.3 From e54538c461e993827d9fbe3b8883d3887f184798 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jun 2023 16:09:29 +0200 Subject: restoreMountNamespace(): Restore the original root directory This is necessary when we're in a chroot environment, where the process root is not the same as the root of the mount namespace (e.g. in nixos-enter). Fixes #7602. --- src/libutil/util.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aa0a154fd..26f9dc8a8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1853,6 +1853,7 @@ void setStackSize(size_t stackSize) #if __linux__ static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; #endif void saveMountNamespace() @@ -1860,10 +1861,11 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); - if (!fd) + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) throw SysError("saving parent mount namespace"); - fdSavedMountNamespace = std::move(fd); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); }); #endif } @@ -1876,9 +1878,16 @@ void restoreMountNamespace() if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); - if (chdir(savedCwd.c_str()) == -1) { - throw SysError("restoring cwd"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); } catch (Error & e) { debug(e.msg()); } -- cgit v1.2.3 From 468add5aa0b5b0a686cd07aa5417817bb8799602 Mon Sep 17 00:00:00 2001 From: Daniel Asaturov <122093031+salpelter@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:09:11 +0400 Subject: Remove dead code (#8504) `filesystem.cc` is the only place where `createSymlink()` is used with three arguments: in the definition of `replaceSymlink()` with three parameters that _is not used at all_. Closes #8495 --- src/libutil/filesystem.cc | 17 +++-------------- src/libutil/util.hh | 6 ++---- 2 files changed, 5 insertions(+), 18 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 56be76ecc..11cc0c0e7 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -63,30 +63,19 @@ std::pair createTempFile(const Path & prefix) return {std::move(fd), tmpl}; } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) +void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } } -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) +void replaceSymlink(const Path & target, const Path & link) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp, mtime); + createSymlink(target, tmp); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 00fcb9b79..b302d6f45 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,14 +256,12 @@ inline Paths createDirs(PathView path) /** * Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional mtime = {}); +void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime = {}); +void replaceSymlink(const Path & target, const Path & link); void renameFile(const Path & src, const Path & dst); -- cgit v1.2.3 From b2247ef4f6b02cf4e666455eb9afc86398fff35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 15 Jun 2023 21:24:14 +0200 Subject: Don't assume the type of string::size_type The code accidentally conflated `std::string::size_type` and `long unsigned int`. This was fine on 64bits machines where they are apparently the same in practice, but not on 32bits. Fix that by using `std::string::size_type` everywhere. --- src/libutil/references.cc | 2 +- src/libutil/references.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 74003584a..7f59b4c09 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -75,7 +75,7 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) : rewrites(rewrites), nextSink(nextSink) { - long unsigned int maxRewriteSize = 0; + std::string::size_type maxRewriteSize = 0; for (auto & [from, to] : rewrites) { assert(from.size() == to.size()); maxRewriteSize = std::max(maxRewriteSize, from.size()); diff --git a/src/libutil/references.hh b/src/libutil/references.hh index ffd730e7b..f0baeffe1 100644 --- a/src/libutil/references.hh +++ b/src/libutil/references.hh @@ -26,7 +26,7 @@ public: struct RewritingSink : Sink { const StringMap rewrites; - long unsigned int maxRewriteSize; + std::string::size_type maxRewriteSize; std::string prev; Sink & nextSink; uint64_t pos = 0; -- cgit v1.2.3 From c8825e9d8c3fa802811f3829d055e3ef9aae90e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 16 Jun 2023 15:19:14 -0400 Subject: Create nlohmann serializers for `std::optional` and use This is somewhat tricky. --- src/libutil/args.cc | 15 ++----- src/libutil/experimental-features.hh | 8 +++- src/libutil/json-utils.cc | 19 +++++++++ src/libutil/json-utils.hh | 78 +++++++++++++++++++++++++++++++----- 4 files changed, 96 insertions(+), 24 deletions(-) create mode 100644 src/libutil/json-utils.cc (limited to 'src/libutil') diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 081dbeb28..3cf3ed9ca 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,10 +1,9 @@ #include "args.hh" #include "hash.hh" +#include "json-utils.hh" #include -#include - namespace nix { void Args::addFlag(Flag && flag_) @@ -247,11 +246,7 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; - // TODO With C++23 use `std::optional::tranform` - if (auto & xp = flag->experimentalFeature) - j["experimental-feature"] = showExperimentalFeature(*xp); - else - j["experimental-feature"] = nullptr; + j["experimental-feature"] = flag->experimentalFeature; flags[name] = std::move(j); } @@ -416,11 +411,7 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); - // TODO With C++23 use `std::optional::tranform` - if (auto xp = command->experimentalFeature()) - cat["experimental-feature"] = showExperimentalFeature(*xp); - else - cat["experimental-feature"] = nullptr; + cat["experimental-feature"] = command->experimentalFeature(); cmds[name] = std::move(j); } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 892c6c371..15ff5e0cd 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -3,7 +3,7 @@ #include "comparator.hh" #include "error.hh" -#include "nlohmann/json_fwd.hpp" +#include "json-utils.hh" #include "types.hh" namespace nix { @@ -93,4 +93,10 @@ public: void to_json(nlohmann::json &, const ExperimentalFeature &); void from_json(const nlohmann::json &, ExperimentalFeature &); +/** + * It is always rendered as a string + */ +template<> +struct json_avoids_null : std::true_type {}; + } diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc new file mode 100644 index 000000000..d7220e71d --- /dev/null +++ b/src/libutil/json-utils.cc @@ -0,0 +1,19 @@ +#include "json-utils.hh" + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +} diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index eb00e954f..5e63c1af4 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -2,21 +2,77 @@ ///@file #include +#include namespace nix { -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +const nlohmann::json * get(const nlohmann::json & map, const std::string & key); + +nlohmann::json * get(nlohmann::json & map, const std::string & key); + +/** + * For `adl_serializer>` below, we need to track what + * types are not already using `null`. Only for them can we use `null` + * to represent `std::nullopt`. + */ +template +struct json_avoids_null; + +/** + * Handle numbers in default impl + */ +template +struct json_avoids_null : std::bool_constant::value> {}; + +template<> +struct json_avoids_null : std::false_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template<> +struct json_avoids_null : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; + +template +struct json_avoids_null> : std::true_type {}; -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; } +namespace nlohmann { + +/** + * This "instance" is widely requested, see + * https://github.com/nlohmann/json/issues/1749, but momentum has stalled + * out. Writing there here in Nix as a stop-gap. + * + * We need to make sure the underlying type does not use `null` for this to + * round trip. We do that with a static assert. + */ +template +struct adl_serializer> { + static std::optional from_json(const json & json) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + return json.is_null() + ? std::nullopt + : std::optional { adl_serializer::from_json(json) }; + } + static void to_json(json & json, std::optional t) { + static_assert( + nix::json_avoids_null::value, + "null is already in use for underlying type's JSON"); + if (t) + adl_serializer::to_json(json, *t); + else + json = nullptr; + } +}; + } -- cgit v1.2.3 From d2ce2e89b178e7e7467cf4c1e572704a4c2ca75e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 May 2023 10:56:59 -0400 Subject: Split `OptionalPathSetting` from `PathSetting` Rather than doing `allowEmpty` as boolean, have separate types and use `std::optional`. This makes it harder to forget the possibility of an empty path. The `build-hook` setting was categorized as a `PathSetting`, but actually it was split into arguments. No good! Now, it is `Setting` which actually reflects what it means and how it is used. Because of the subtyping, we now also have support for `Setting>` in general. I imagine this can be used to clean up many more settings also. --- src/libutil/abstract-setting-to-json.hh | 1 + src/libutil/config-impl.hh | 61 ++++++++++++++++++++++++++++++- src/libutil/config.cc | 63 ++++++++++++--------------------- src/libutil/config.hh | 31 +++++++++++++--- 4 files changed, 111 insertions(+), 45 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 7b6c3fcb5..d506dfb74 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -3,6 +3,7 @@ #include #include "config.hh" +#include "json-utils.hh" namespace nix { template diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b6cae5ec3..14822050e 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -50,8 +50,11 @@ template<> void BaseSetting>::appendOrSet(std::set template void BaseSetting::appendOrSet(T && newValue, bool append) { - static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + static_assert( + !trait::appendable, + "using default `appendOrSet` implementation with an appendable type"); assert(!append); + value = std::move(newValue); } @@ -68,4 +71,60 @@ void BaseSetting::set(const std::string & str, bool append) } } +template<> void BaseSetting::convertToArg(Args & args, const std::string & category); + +template +void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .description = fmt("Set the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, + }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .description = fmt("Append to the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, + }); +} + +#define DECLARE_CONFIG_SERIALISER(TY) \ + template<> TY BaseSetting< TY >::parse(const std::string & str) const; \ + template<> std::string BaseSetting< TY >::to_string() const; + +DECLARE_CONFIG_SERIALISER(std::string) +DECLARE_CONFIG_SERIALISER(std::optional) +DECLARE_CONFIG_SERIALISER(bool) +DECLARE_CONFIG_SERIALISER(Strings) +DECLARE_CONFIG_SERIALISER(StringSet) +DECLARE_CONFIG_SERIALISER(StringMap) +DECLARE_CONFIG_SERIALISER(std::set) + +template +T BaseSetting::parse(const std::string & str) const +{ + static_assert(std::is_integral::value, "Integer required."); + + if (auto n = string2Int(str)) + return *n; + else + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template +std::string BaseSetting::to_string() const +{ + static_assert(std::is_integral::value, "Integer required."); + + return std::to_string(value); +} + } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 085a884dc..38d406e8a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -219,29 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template -void BaseSetting::convertToArg(Args & args, const std::string & category) -{ - args.addFlag({ - .longName = name, - .description = fmt("Set the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s); }}, - .experimentalFeature = experimentalFeature, - }); - - if (isAppendable()) - args.addFlag({ - .longName = "extra-" + name, - .description = fmt("Append to the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s, true); }}, - .experimentalFeature = experimentalFeature, - }); -} - template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -252,21 +229,17 @@ template<> std::string BaseSetting::to_string() const return value; } -template -T BaseSetting::parse(const std::string & str) const +template<> std::optional BaseSetting>::parse(const std::string & str) const { - static_assert(std::is_integral::value, "Integer required."); - if (auto n = string2Int(str)) - return *n; + if (str == "") + return std::nullopt; else - throw UsageError("setting '%s' has invalid value '%s'", name, str); + return { str }; } -template -std::string BaseSetting::to_string() const +template<> std::string BaseSetting>::to_string() const { - static_assert(std::is_integral::value, "Integer required."); - return std::to_string(value); + return value ? *value : ""; } template<> bool BaseSetting::parse(const std::string & str) const @@ -403,17 +376,27 @@ template class BaseSetting; template class BaseSetting; template class BaseSetting>; -Path PathSetting::parse(const std::string & str) const +static Path parsePath(const AbstractSetting & s, const std::string & str) { - if (str == "") { - if (allowEmpty) - return ""; - else - throw UsageError("setting '%s' cannot be empty", name); - } else + if (str == "") + throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); + else return canonPath(str); } +Path PathSetting::parse(const std::string & str) const +{ + return parsePath(*this, str); +} + +std::optional OptionalPathSetting::parse(const std::string & str) const +{ + if (str == "") + return std::nullopt; + else + return parsePath(*this, str); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 2675baed7..cc8532587 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -353,21 +353,20 @@ public: /** * A special setting for Paths. These are automatically canonicalised * (e.g. "/foo//bar/" becomes "/foo/bar"). + * + * It is mandatory to specify a path; i.e. the empty string is not + * permitted. */ class PathSetting : public BaseSetting { - bool allowEmpty; - public: PathSetting(Config * options, - bool allowEmpty, const Path & def, const std::string & name, const std::string & description, const std::set & aliases = {}) : BaseSetting(def, true, name, description, aliases) - , allowEmpty(allowEmpty) { options->addSetting(this); } @@ -379,6 +378,30 @@ public: void operator =(const Path & v) { this->assign(v); } }; +/** + * Like `PathSetting`, but the absence of a path is also allowed. + * + * `std::optional` is used instead of the empty string for clarity. + */ +class OptionalPathSetting : public BaseSetting> +{ +public: + + OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting>(def, true, name, description, aliases) + { + options->addSetting(this); + } + + std::optional parse(const std::string & str) const override; + + void operator =(const std::optional & v) { this->assign(v); } +}; + struct GlobalConfig : public AbstractConfig { typedef std::vector ConfigRegistrations; -- cgit v1.2.3 From 469d06f9bc9b2543f48d8e633e47f9344fba35ab Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 8 Mar 2022 21:53:26 +0000 Subject: Split out worker protocol template definitions from declarations This is generally a fine practice: Putting implementations in headers makes them harder to read and slows compilation. Unfortunately it is necessary for templates, but we can ameliorate that by putting them in a separate header. Only files which need to instantiate those templates will need to include the header with the implementation; the rest can just include the declaration. This is now documenting in the contributing guide. Also, it just happens that these polymorphic serializers are the protocol agnostic ones. (Worker and serve protocol have the same logic for these container types.) This means by doing this general template cleanup, we are also getting a head start on better indicating which code is protocol-specific and which code is shared between protocols. --- src/libutil/config-impl.hh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b6cae5ec3..d9726a907 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -4,6 +4,9 @@ * * Template implementations (as opposed to mere declarations). * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + * * One only needs to include this when one is declaring a * `BaseClass` setting, or as derived class of such an * instantiation. -- cgit v1.2.3 From 6ae35534b7b6e10a26a0f2b2a0e37d7f7cfe47dd Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Tue, 20 Jun 2023 10:34:09 +0100 Subject: Support opening local store with database on read-only filesystem (#8356) Previously it was not possible to open a local store when its database is on a read-only filesystem. Obviously a store on a read-only filesystem cannot be modified, but it would still be useful to be able to query it. This change adds a new read-only setting to LocalStore. When set to true, Nix will skip operations that fail when the database is on a read-only filesystem (acquiring big-lock, schema migration, etc), and the store database will be opened in immutable mode. Co-authored-by: Ben Radford Co-authored-by: cidkidnix Co-authored-by: Dylan Green <67574902+cidkidnix@users.noreply.github.com> Co-authored-by: John Ericson Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 9 ++++++++- src/libutil/experimental-features.hh | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index c4642d333..7c4112d32 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -221,6 +221,13 @@ constexpr std::array xpFeatureDetails = {{ Allow parsing of timestamps in builtins.fromTOML. )", }, + { + .tag = Xp::ReadOnlyLocalStore, + .name = "read-only-local-store", + .description = R"( + Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 892c6c371..507b0cc06 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -31,6 +31,7 @@ enum struct ExperimentalFeature DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, + ReadOnlyLocalStore, }; /** -- cgit v1.2.3