From 176911102ce2c0be06bbfed9099f364d71c3c679 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 13 Sep 2021 11:57:25 -0600 Subject: printEnvPosChain --- src/libutil/error.hh | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index ff58d3e00..b6670c8b2 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -100,6 +100,12 @@ struct ErrPos { } }; +std::optional getCodeLines(const ErrPos & errPos); +void printCodeLines(std::ostream & out, + const string & prefix, + const ErrPos & errPos, + const LinesOfCode & loc); + struct Trace { std::optional pos; hintformat hint; -- cgit v1.2.3 From b4a59a5eec5bdb94ee2bbc8365f024d5787abd60 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 22 Dec 2021 15:38:49 -0700 Subject: DebugStackTracker class in one place --- src/libutil/error.cc | 2 ++ src/libutil/logging.hh | 1 + 2 files changed, 3 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 203d79087..c2b9d2707 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -221,6 +221,8 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { + std::cout << "showErrorInfo showTrace: " << showTrace << std::endl; + std::string prefix; switch (einfo.level) { case Verbosity::lvlError: { diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 96ad69790..f10a9af38 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -38,6 +38,7 @@ typedef uint64_t ActivityId; struct LoggerSettings : Config { Setting showTrace{ + // this, false, "show-trace", this, false, "show-trace", R"( Where Nix should print out a stack trace in case of Nix -- cgit v1.2.3 From bc20e54e0044e08c68a7af1f1e12d001baba8a74 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Wed, 22 Dec 2021 19:40:08 -0700 Subject: stack traces basically working --- src/libutil/error.hh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index b6670c8b2..06301f709 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -106,6 +106,9 @@ void printCodeLines(std::ostream & out, const ErrPos & errPos, const LinesOfCode & loc); +void printAtPos(const ErrPos & pos, std::ostream & out); + + struct Trace { std::optional pos; hintformat hint; -- cgit v1.2.3 From 4610e02d04c9f41ac355d2ca6a27d3a631ffefc6 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 27 Dec 2021 18:12:46 -0700 Subject: remove debug code --- src/libutil/error.cc | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c2b9d2707..203d79087 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -221,8 +221,6 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { - std::cout << "showErrorInfo showTrace: " << showTrace << std::endl; - std::string prefix; switch (einfo.level) { case Verbosity::lvlError: { -- cgit v1.2.3 From 5954cbf3e9dca0e3b84e4bf2def74abb3d6f80cd Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 27 Dec 2021 18:29:55 -0700 Subject: more cleanup --- src/libutil/logging.hh | 1 - 1 file changed, 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index f10a9af38..96ad69790 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -38,7 +38,6 @@ typedef uint64_t ActivityId; struct LoggerSettings : Config { Setting showTrace{ - // this, false, "show-trace", this, false, "show-trace", R"( Where Nix should print out a stack trace in case of Nix -- cgit v1.2.3 From e6d07e0d89d964cde22894fca57a95177c085c8d Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 18 Mar 2022 00:58:09 +0100 Subject: Refactor to use more traces and less string manipulations --- src/libutil/error.cc | 23 +++++++++++++++-------- src/libutil/error.hh | 10 ++++++++-- 2 files changed, 23 insertions(+), 10 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index dcd2f82a5..134b99893 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -185,15 +185,15 @@ void printAtPos(const ErrPos & pos, std::ostream & out) if (pos) { switch (pos.origin) { case foFile: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); + out << fmt(ANSI_BLUE " at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); break; } case foString: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); + out << fmt(ANSI_BLUE " at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } case foStdin: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); + out << fmt(ANSI_BLUE " at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } default: @@ -269,7 +269,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << einfo.msg << "\n"; if (einfo.errPos.has_value() && *einfo.errPos) { - oss << "\n"; printAtPos(*einfo.errPos, oss); auto loc = getCodeLines(*einfo.errPos); @@ -278,26 +277,34 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s if (loc.has_value()) { oss << "\n"; printCodeLines(oss, "", *einfo.errPos, *loc); - oss << "\n"; } + oss << "\n"; } // traces - if (showTrace && !einfo.traces.empty()) { + if (!einfo.traces.empty()) { + unsigned int count = 0; for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { + if (!showTrace && count > 3) { + oss << "\n" << "(truncated)" << "\n"; + break; + } + + if (iter->hint.str().empty()) continue; + count++; oss << "\n" << "… " << iter->hint.str() << "\n"; if (iter->pos.has_value() && (*iter->pos)) { + count++; auto pos = iter->pos.value(); - oss << "\n"; printAtPos(pos, oss); auto loc = getCodeLines(pos); if (loc.has_value()) { oss << "\n"; printCodeLines(oss, "", pos, *loc); - oss << "\n"; } + oss << "\n"; } } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d55e1d701..ad29f8d2a 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -130,6 +130,8 @@ protected: public: unsigned int status = 1; // exit status + BaseError(const BaseError &) = default; + template BaseError(unsigned int status, const Args & ... args) : err { .level = lvlError, .msg = hintfmt(args...) } @@ -165,10 +167,14 @@ public: const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } + void pushTrace(Trace trace) { + err.traces.push_front(trace); + } + template - BaseError & addTrace(std::optional e, const std::string & fs, const Args & ... args) + BaseError & addTrace(std::optional e, const std::string_view & fs, const Args & ... args) { - return addTrace(e, hintfmt(fs, args...)); + return addTrace(e, hintfmt(std::string(fs), args...)); } BaseError & addTrace(std::optional e, hintformat hint); -- cgit v1.2.3 From 963b8aa39b169bf5c054449ddce39d60faacf298 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Fri, 18 Mar 2022 23:17:50 +0100 Subject: Explain current error trace impl --- src/libutil/error.cc | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index ae0bbc06f..c66dfa112 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -288,7 +288,100 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s "?" << std::endl; } - // traces + /* + * Traces + * ------ + * + * The semantics of traces is a bit weird. We have only one option to + * print them and to make them verbose (--show-trace). In the code they + * are always collected, but they are not printed by default. The code + * also collects more traces when the option is on. This means that there + * is no way to print the simplified traces at all. + * + * I (layus) designed the code to attach positions to a restricted set of + * messages. This means that we have a lot of traces with no position at + * all, including most of the base error messages. For example "type + * error: found a string while a set was expected" has no position, but + * will come with several traces detailing it's precise relation to the + * closest know position. This makes erroring without printing traces + * quite useless. + * + * This is why I introduced the idea to always print a few traces on + * error. The number 3 is quite arbitrary, and was selected so as not to + * clutter the console on error. For the same reason, a trace with an + * error position takes more space, and counts as two traces towards the + * limit. + * + * The rest is truncated, unless --show-trace is passed. This preserves + * the same bad semantics of --show-trace to both show the trace and + * augment it with new data. Not too sure what is the best course of + * action. + * + * The issue is that it is fundamentally hard to provide a trace for a + * lazy language. The trace will only cover the current spine of the + * evaluation, missing things that have been evaluated before. For + * example, most type errors are hard to inspect because there is not + * trace for the faulty value. These errors should really print the faulty + * value itself. + * + * In function calls, the --show-trace flag triggers extra traces for each + * function invocation. These work as scopes, allowing to follow the + * current spine of the evaluation graph. Without that flag, the error + * trace should restrict itself to a restricted prefix of that trace, + * until the first scope. If we ever get to such a precise error + * reporting, there would be no need to add an arbitrary limit here. We + * could always print the full trace, and it would just be small without + * the flag. + * + * One idea I had is for XxxError.addTrace() to perform nothing if one + * scope has already been traced. Alternatively, we could stop here when + * we encounter such a scope instead of after an arbitrary number of + * traces. This however requires to augment traces with the notion of + * "scope". + * + * This is particularly visible in code like evalAttrs(...) where we have + * to make a decision between the two following options. + * + * ``` long traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, const std::string_view & errorCtx) + * { + * try { + * e->eval(*this, env, v); + * if (v.type() != nAttrs) + * throwTypeError("value is %1% while a set was expected", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * ``` short traces + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, const std::string_view & errorCtx) + * { + * e->eval(*this, env, v); + * try { + * if (v.type() != nAttrs) + * throwTypeError("value is %1% while a set was expected", v); + * } catch (Error & e) { + * e.addTrace(pos, errorCtx); + * throw; + * } + * } + * ``` + * + * The second example can be rewritten more concisely, but kept in this + * form to highlight the symmetry. The first option adds more information, + * because whatever caused an error down the line, in the generic eval + * function, will get annotated with the code location that uses and + * required it. The second option is less verbose, but does not provide + * any context at all as to where and why a failing value was required. + * + * Scopes would fix that, by adding context only when --show-trace is + * passed, and keeping the trace terse otherwise. + * + */ + if (!einfo.traces.empty()) { unsigned int count = 0; for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { -- cgit v1.2.3 From 75b62e52600a44b42693944b50638bf580a2c86e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jun 2020 17:54:16 +0000 Subject: Avoid `fmt` when constructor already does it There is a correctnes issue here, but #3724 will fix that. This is just a cleanup for brevity's sake. --- src/libutil/util.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index c075a14b4..d4379a0cf 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1062,7 +1062,7 @@ std::string runProgram(Path program, bool searchPath, const Strings & args, auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); return res.second; } @@ -1190,7 +1190,7 @@ void runProgram2(const RunOptions & options) if (source) promise.get_future().get(); if (status) - throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); } -- cgit v1.2.3 From ebf2fd76b106d5eb8f45ccce0615653108bb99bc Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 20 Apr 2022 15:41:01 +0200 Subject: Add custom to_json and from_json functions for ExperimentalFeature nix show-config --json was serializing experimental features as ints. nlohmann::json will automatically use these definitions to serialize and deserialize ExperimentalFeatures. Strictly, we don't use the from_json instance yet, it's provided for completeness and hopefully future use. --- src/libutil/experimental-features.cc | 14 ++++++++++++++ src/libutil/experimental-features.hh | 7 +++++++ 2 files changed, 21 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e033a4116..df37edf57 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -58,4 +58,18 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu return str << showExperimentalFeature(feature); } +void to_json(nlohmann::json& j, const ExperimentalFeature& feature) { + j = showExperimentalFeature(feature); +} + +void from_json(const nlohmann::json& j, ExperimentalFeature& feature) { + const std::string input = j; + const auto parsed = parseExperimentalFeature(input); + + if (parsed.has_value()) + feature = *parsed; + else + throw Error("Unknown experimental feature '%s' in JSON input", input); +} + } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 266e41a22..a6d080094 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -51,4 +51,11 @@ public: MissingExperimentalFeature(ExperimentalFeature); }; +/** + * Semi-magic conversion to and from json. + * See the nlohmann/json readme for more details. + */ +void to_json(nlohmann::json&, const ExperimentalFeature&); +void from_json(const nlohmann::json&, ExperimentalFeature&); + } -- cgit v1.2.3 From f05e1f6fbb8a760f23a7af16b065078df6588acf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Apr 2022 11:58:40 +0200 Subject: Move hiliteMatches into a separate header This is mostly so that we don't #include everywhere (which adds quite a bit of compilation time). --- src/libutil/fmt.cc | 46 ---------------------------------------------- src/libutil/fmt.hh | 12 ------------ src/libutil/hilite.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/libutil/hilite.hh | 20 ++++++++++++++++++++ 4 files changed, 64 insertions(+), 58 deletions(-) delete mode 100644 src/libutil/fmt.cc create mode 100644 src/libutil/hilite.cc create mode 100644 src/libutil/hilite.hh (limited to 'src/libutil') diff --git a/src/libutil/fmt.cc b/src/libutil/fmt.cc deleted file mode 100644 index 3dd93d73e..000000000 --- a/src/libutil/fmt.cc +++ /dev/null @@ -1,46 +0,0 @@ -#include "fmt.hh" - -#include - -namespace nix { - -std::string hiliteMatches( - std::string_view s, - std::vector matches, - std::string_view prefix, - std::string_view postfix) -{ - // Avoid copy on zero matches - if (matches.size() == 0) - return (std::string) s; - - std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { - return a.position() < b.position(); - }); - - std::string out; - ssize_t last_end = 0; - - for (auto it = matches.begin(); it != matches.end();) { - auto m = *it; - size_t start = m.position(); - out.append(s.substr(last_end, m.position() - last_end)); - // Merge continous matches - ssize_t end = start + m.length(); - while (++it != matches.end() && (*it).position() <= end) { - auto n = *it; - ssize_t nend = start + (n.position() - start + n.length()); - if (nend > end) - end = nend; - } - out.append(prefix); - out.append(s.substr(start, end - start)); - out.append(postfix); - last_end = end; - } - - out.append(s.substr(last_end)); - return out; -} - -} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 0821b3b74..7664e5c04 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -2,7 +2,6 @@ #include #include -#include #include "ansicolor.hh" @@ -155,15 +154,4 @@ inline hintformat hintfmt(std::string plain_string) return hintfmt("%s", normaltxt(plain_string)); } -/* Highlight all the given matches in the given string `s` by wrapping - them between `prefix` and `postfix`. - - If some matches overlap, then their union will be wrapped rather - than the individual matches. */ -std::string hiliteMatches( - std::string_view s, - std::vector matches, - std::string_view prefix, - std::string_view postfix); - } diff --git a/src/libutil/hilite.cc b/src/libutil/hilite.cc new file mode 100644 index 000000000..a5991ca39 --- /dev/null +++ b/src/libutil/hilite.cc @@ -0,0 +1,44 @@ +#include "hilite.hh" + +namespace nix { + +std::string hiliteMatches( + std::string_view s, + std::vector matches, + std::string_view prefix, + std::string_view postfix) +{ + // Avoid copy on zero matches + if (matches.size() == 0) + return (std::string) s; + + std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { + return a.position() < b.position(); + }); + + std::string out; + ssize_t last_end = 0; + + for (auto it = matches.begin(); it != matches.end();) { + auto m = *it; + size_t start = m.position(); + out.append(s.substr(last_end, m.position() - last_end)); + // Merge continous matches + ssize_t end = start + m.length(); + while (++it != matches.end() && (*it).position() <= end) { + auto n = *it; + ssize_t nend = start + (n.position() - start + n.length()); + if (nend > end) + end = nend; + } + out.append(prefix); + out.append(s.substr(start, end - start)); + out.append(postfix); + last_end = end; + } + + out.append(s.substr(last_end)); + return out; +} + +} diff --git a/src/libutil/hilite.hh b/src/libutil/hilite.hh new file mode 100644 index 000000000..f8bdbfc55 --- /dev/null +++ b/src/libutil/hilite.hh @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include + +namespace nix { + +/* Highlight all the given matches in the given string `s` by wrapping + them between `prefix` and `postfix`. + + If some matches overlap, then their union will be wrapped rather + than the individual matches. */ +std::string hiliteMatches( + std::string_view s, + std::vector matches, + std::string_view prefix, + std::string_view postfix); + +} -- cgit v1.2.3 From f1eee873ea064e8f94369bdfe2557c18ba35ccc9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Apr 2022 13:00:24 +0200 Subject: Fix fmt test --- src/libutil/tests/fmt.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/tests/fmt.cc b/src/libutil/tests/fmt.cc index 33772162c..1ff5980d5 100644 --- a/src/libutil/tests/fmt.cc +++ b/src/libutil/tests/fmt.cc @@ -1,9 +1,7 @@ -#include "fmt.hh" +#include "hilite.hh" #include -#include - namespace nix { /* ----------- tests for fmt.hh -------------------------------------------------*/ -- cgit v1.2.3 From 3b9d31b88c95e591c28f3a7423f83c40b9788781 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Apr 2022 13:00:50 +0200 Subject: Rename fmt test -> hilte --- src/libutil/tests/fmt.cc | 66 --------------------------------------------- src/libutil/tests/hilite.cc | 66 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 66 deletions(-) delete mode 100644 src/libutil/tests/fmt.cc create mode 100644 src/libutil/tests/hilite.cc (limited to 'src/libutil') diff --git a/src/libutil/tests/fmt.cc b/src/libutil/tests/fmt.cc deleted file mode 100644 index 1ff5980d5..000000000 --- a/src/libutil/tests/fmt.cc +++ /dev/null @@ -1,66 +0,0 @@ -#include "hilite.hh" - -#include - -namespace nix { -/* ----------- tests for fmt.hh -------------------------------------------------*/ - - TEST(hiliteMatches, noHighlight) { - ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector(), "(", ")").c_str(), "Hello, world!"); - } - - TEST(hiliteMatches, simpleHighlight) { - std::string str = "Hello, world!"; - std::regex re = std::regex("world"); - auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); - ASSERT_STREQ( - hiliteMatches(str, matches, "(", ")").c_str(), - "Hello, (world)!" - ); - } - - TEST(hiliteMatches, multipleMatches) { - std::string str = "Hello, world, world, world, world, world, world, Hello!"; - std::regex re = std::regex("world"); - auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); - ASSERT_STREQ( - hiliteMatches(str, matches, "(", ")").c_str(), - "Hello, (world), (world), (world), (world), (world), (world), Hello!" - ); - } - - TEST(hiliteMatches, overlappingMatches) { - std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!"; - std::regex re = std::regex("Hello, world"); - std::regex re2 = std::regex("world, Hello"); - auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); - for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) { - v.push_back(*it); - } - ASSERT_STREQ( - hiliteMatches(str, v, "(", ")").c_str(), - "(world, Hello, world, Hello, world, Hello, world, Hello, world)!" - ); - } - - TEST(hiliteMatches, complexOverlappingMatches) { - std::string str = "legacyPackages.x86_64-linux.git-crypt"; - std::vector regexes = { - std::regex("t-cry"), - std::regex("ux\\.git-cry"), - std::regex("git-c"), - std::regex("pt"), - }; - std::vector matches; - for(auto regex : regexes) - { - for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { - matches.push_back(*it); - } - } - ASSERT_STREQ( - hiliteMatches(str, matches, "(", ")").c_str(), - "legacyPackages.x86_64-lin(ux.git-crypt)" - ); - } -} diff --git a/src/libutil/tests/hilite.cc b/src/libutil/tests/hilite.cc new file mode 100644 index 000000000..1ff5980d5 --- /dev/null +++ b/src/libutil/tests/hilite.cc @@ -0,0 +1,66 @@ +#include "hilite.hh" + +#include + +namespace nix { +/* ----------- tests for fmt.hh -------------------------------------------------*/ + + TEST(hiliteMatches, noHighlight) { + ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector(), "(", ")").c_str(), "Hello, world!"); + } + + TEST(hiliteMatches, simpleHighlight) { + std::string str = "Hello, world!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world)!" + ); + } + + TEST(hiliteMatches, multipleMatches) { + std::string str = "Hello, world, world, world, world, world, world, Hello!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world), (world), (world), (world), (world), (world), Hello!" + ); + } + + TEST(hiliteMatches, overlappingMatches) { + std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!"; + std::regex re = std::regex("Hello, world"); + std::regex re2 = std::regex("world, Hello"); + auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) { + v.push_back(*it); + } + ASSERT_STREQ( + hiliteMatches(str, v, "(", ")").c_str(), + "(world, Hello, world, Hello, world, Hello, world, Hello, world)!" + ); + } + + TEST(hiliteMatches, complexOverlappingMatches) { + std::string str = "legacyPackages.x86_64-linux.git-crypt"; + std::vector regexes = { + std::regex("t-cry"), + std::regex("ux\\.git-cry"), + std::regex("git-c"), + std::regex("pt"), + }; + std::vector matches; + for(auto regex : regexes) + { + for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { + matches.push_back(*it); + } + } + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "legacyPackages.x86_64-lin(ux.git-crypt)" + ); + } +} -- cgit v1.2.3 From 6526d1676ba5a645f65d751e7529ccd273579017 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 4 Mar 2022 19:31:59 +0100 Subject: replace most Pos objects/ptrs with indexes into a position table Pos objects are somewhat wasteful as they duplicate the origin file name and input type for each object. on files that produce more than one Pos when parsed this a sizeable waste of memory (one pointer per Pos). the same goes for ptr on 64 bit machines: parsing enough source to require 8 bytes to locate a position would need at least 8GB of input and 64GB of expression memory. it's not likely that we'll hit that any time soon, so we can use a uint32_t index to locate positions instead. --- src/libutil/types.hh | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 00ba567c6..22bc2b8dd 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -102,4 +103,55 @@ public: Ptr operator->() const { return Ptr(**this); } }; +/* Provides an indexable container like vector<> with memory overhead + guarantees like list<> by allocating storage in chunks of ChunkSize + elements instead of using a contiguous memory allocation like vector<> + does. Not using a single vector that is resized reduces memory overhead + on large data sets by on average (growth factor)/2, mostly + eliminates copies within the vector during resizing, and provides stable + references to its elements. */ +template +class ChunkedVector { +private: + uint32_t size_ = 0; + std::vector> chunks; + + /* keep this out of the ::add hot path */ + [[gnu::noinline]] + auto & addChunk() + { + if (size_ >= std::numeric_limits::max() - ChunkSize) + abort(); + chunks.emplace_back(); + chunks.back().reserve(ChunkSize); + return chunks.back(); + } + +public: + ChunkedVector(uint32_t reserve) + { + chunks.reserve(reserve); + addChunk(); + } + + uint32_t size() const { return size_; } + + std::pair add(T value) + { + const auto idx = size_++; + auto & chunk = [&] () -> auto & { + if (auto & back = chunks.back(); back.size() < ChunkSize) + return back; + return addChunk(); + }(); + auto & result = chunk.emplace_back(std::move(value)); + return {result, idx}; + } + + const T & operator[](uint32_t idx) const + { + return chunks[idx / ChunkSize][idx % ChunkSize]; + } +}; + } -- cgit v1.2.3 From 00a32802328b58daa7af48ccac60f6154ef05639 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 5 Mar 2022 17:31:50 +0100 Subject: don't use Symbol in Pos to represent a path PosTable deduplicates origin information, so using symbols for paths is no longer necessary. moving away from path Symbols also reduces the usage of symbols for things that are not keys in attribute sets, which will become important in the future when we turn symbols into indices as well. --- src/libutil/error.hh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 348018f57..f4706e3ed 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -87,11 +87,7 @@ struct ErrPos { origin = pos.origin; line = pos.line; column = pos.column; - // is file symbol null? - if (pos.file.set()) - file = pos.file; - else - file = ""; + file = pos.file; return *this; } -- cgit v1.2.3 From 8775be33931ec3b1cad97035ff3d5370a97178a1 Mon Sep 17 00:00:00 2001 From: pennae Date: Sat, 5 Mar 2022 14:40:24 +0100 Subject: store Symbols in a table as well, like positions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this slightly increases the amount of memory used for any given symbol, but this increase is more than made up for if the symbol is referenced more than once in the EvalState that holds it. on average every symbol should be referenced at least twice (once to introduce a binding, once to use it), so we expect no increase in memory on average. symbol tables are limited to 2³² entries like position tables, and similar arguments apply to why overflow is not likely: 2³² symbols would require as many string instances (at 24 bytes each) and map entries (at 24 bytes or more each, assuming that the map holds on average at most one item per bucket as the docs say). a full symbol table would require at least 192GB of memory just for symbols, which is well out of reach. (an ofborg eval of nixpks today creates less than a million symbols!) --- src/libutil/types.hh | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 22bc2b8dd..bd1dd8bee 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -152,6 +152,14 @@ public: { return chunks[idx / ChunkSize][idx % ChunkSize]; } + + template + void forEach(Fn fn) const + { + for (const auto & c : chunks) + for (const auto & e : c) + fn(e); + } }; } -- cgit v1.2.3 From 8adaa6acb5a513e010d262386271ef39c418ea7f Mon Sep 17 00:00:00 2001 From: pennae Date: Tue, 5 Apr 2022 18:37:38 +0200 Subject: remove pos it's no longer needed now that positions aren't really pointers any more. --- src/libutil/ref.hh | 43 ------------------------------------------- 1 file changed, 43 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 347b81f73..f9578afc7 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -99,47 +99,4 @@ make_ref(Args&&... args) return ref(p); } - -/* A non-nullable pointer. - This is similar to a C++ "& reference", but mutable. - This is similar to ref but backed by a regular pointer instead of a smart pointer. - */ -template -class ptr { -private: - T * p; - -public: - ptr(const ptr & r) - : p(r.p) - { } - - explicit ptr(T * p) - : p(p) - { - if (!p) - throw std::invalid_argument("null pointer cast to ptr"); - } - - T* operator ->() const - { - return &*p; - } - - T& operator *() const - { - return *p; - } - - bool operator == (const ptr & other) const - { - return p == other.p; - } - - bool operator != (const ptr & other) const - { - return p != other.p; - } -}; - } -- cgit v1.2.3 From 7ca6fbc8caf30d0f8b0dca9a294022d3e37c4082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 22 Apr 2022 10:01:02 +0200 Subject: Move ChunkedVector to its own header --- src/libutil/chunked-vector.hh | 68 +++++++++++++++++++++++++++++++++++++++++++ src/libutil/types.hh | 59 ------------------------------------- 2 files changed, 68 insertions(+), 59 deletions(-) create mode 100644 src/libutil/chunked-vector.hh (limited to 'src/libutil') diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh new file mode 100644 index 000000000..f15af9cd7 --- /dev/null +++ b/src/libutil/chunked-vector.hh @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include + +namespace nix { + +/* Provides an indexable container like vector<> with memory overhead + guarantees like list<> by allocating storage in chunks of ChunkSize + elements instead of using a contiguous memory allocation like vector<> + does. Not using a single vector that is resized reduces memory overhead + on large data sets by on average (growth factor)/2, mostly + eliminates copies within the vector during resizing, and provides stable + references to its elements. */ +template +class ChunkedVector { +private: + uint32_t size_ = 0; + std::vector> chunks; + + /* keep this out of the ::add hot path */ + [[gnu::noinline]] + auto & addChunk() + { + if (size_ >= std::numeric_limits::max() - ChunkSize) + abort(); + chunks.emplace_back(); + chunks.back().reserve(ChunkSize); + return chunks.back(); + } + +public: + ChunkedVector(uint32_t reserve) + { + chunks.reserve(reserve); + addChunk(); + } + + uint32_t size() const { return size_; } + + std::pair add(T value) + { + const auto idx = size_++; + auto & chunk = [&] () -> auto & { + if (auto & back = chunks.back(); back.size() < ChunkSize) + return back; + return addChunk(); + }(); + auto & result = chunk.emplace_back(std::move(value)); + return {result, idx}; + } + + const T & operator[](uint32_t idx) const + { + return chunks[idx / ChunkSize][idx % ChunkSize]; + } + + template + void forEach(Fn fn) const + { + for (const auto & c : chunks) + for (const auto & e : c) + fn(e); + } +}; +} diff --git a/src/libutil/types.hh b/src/libutil/types.hh index bd1dd8bee..6bcbd7e1d 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -103,63 +103,4 @@ public: Ptr operator->() const { return Ptr(**this); } }; -/* Provides an indexable container like vector<> with memory overhead - guarantees like list<> by allocating storage in chunks of ChunkSize - elements instead of using a contiguous memory allocation like vector<> - does. Not using a single vector that is resized reduces memory overhead - on large data sets by on average (growth factor)/2, mostly - eliminates copies within the vector during resizing, and provides stable - references to its elements. */ -template -class ChunkedVector { -private: - uint32_t size_ = 0; - std::vector> chunks; - - /* keep this out of the ::add hot path */ - [[gnu::noinline]] - auto & addChunk() - { - if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); - chunks.emplace_back(); - chunks.back().reserve(ChunkSize); - return chunks.back(); - } - -public: - ChunkedVector(uint32_t reserve) - { - chunks.reserve(reserve); - addChunk(); - } - - uint32_t size() const { return size_; } - - std::pair add(T value) - { - const auto idx = size_++; - auto & chunk = [&] () -> auto & { - if (auto & back = chunks.back(); back.size() < ChunkSize) - return back; - return addChunk(); - }(); - auto & result = chunk.emplace_back(std::move(value)); - return {result, idx}; - } - - const T & operator[](uint32_t idx) const - { - return chunks[idx / ChunkSize][idx % ChunkSize]; - } - - template - void forEach(Fn fn) const - { - for (const auto & c : chunks) - for (const auto & e : c) - fn(e); - } -}; - } -- cgit v1.2.3 From 484badfa096db4c001f66eccbe00b70471f2e767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 22 Apr 2022 10:01:10 +0200 Subject: Add some tests for ChunkedVector --- src/libutil/tests/chunked-vector.cc | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 src/libutil/tests/chunked-vector.cc (limited to 'src/libutil') diff --git a/src/libutil/tests/chunked-vector.cc b/src/libutil/tests/chunked-vector.cc new file mode 100644 index 000000000..868d11f6f --- /dev/null +++ b/src/libutil/tests/chunked-vector.cc @@ -0,0 +1,54 @@ +#include "chunked-vector.hh" + +#include + +namespace nix { + TEST(ChunkedVector, InitEmpty) { + auto v = ChunkedVector(100); + ASSERT_EQ(v.size(), 0); + } + + TEST(ChunkedVector, GrowsCorrectly) { + auto v = ChunkedVector(100); + for (auto i = 1; i < 20; i++) { + v.add(i); + ASSERT_EQ(v.size(), i); + } + } + + TEST(ChunkedVector, AddAndGet) { + auto v = ChunkedVector(100); + for (auto i = 1; i < 20; i++) { + auto [i2, idx] = v.add(i); + auto & i3 = v[idx]; + ASSERT_EQ(i, i2); + ASSERT_EQ(&i2, &i3); + } + } + + TEST(ChunkedVector, ForEach) { + auto v = ChunkedVector(100); + for (auto i = 1; i < 20; i++) { + v.add(i); + } + int count = 0; + v.forEach([&count](int elt) { + count++; + }); + ASSERT_EQ(count, v.size()); + } + + TEST(ChunkedVector, OverflowOK) { + // Similar to the AddAndGet, but intentionnally use a small + // initial ChunkedVector to force it to overflow + auto v = ChunkedVector(2); + for (auto i = 1; i < 20; i++) { + auto [i2, idx] = v.add(i); + auto & i3 = v[idx]; + ASSERT_EQ(i, i2); + ASSERT_EQ(&i2, &i3); + } + } + +} + -- cgit v1.2.3 From 7b889f31eac512c548fe56a73cd57d00d4fb89c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 22 Apr 2022 10:56:01 +0200 Subject: Fix the darwin build Looks like the auto-merge is indeed quite broken and merges even when the CI fails --- src/libutil/chunked-vector.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index f15af9cd7..0a4f0b400 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include #include -- cgit v1.2.3 From 5ef88457b8bdef957fcbb3cd0e7740595fad1949 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 28 Apr 2022 13:00:24 +0200 Subject: Better document error location indent --- src/libutil/error.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c66dfa112..7dc8c1941 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -180,20 +180,24 @@ void printCodeLines(std::ostream & out, } } +// Enough indent to align with with the `... ` +// prepended to each element of the trace +#define ELLIPSIS_INDENT " " + void printAtPos(const ErrPos & pos, std::ostream & out) { if (pos) { switch (pos.origin) { case foFile: { - out << fmt(ANSI_BLUE " at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); + out << fmt(ELLIPSIS_INDENT "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); break; } case foString: { - out << fmt(ANSI_BLUE " at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); + out << fmt(ELLIPSIS_INDENT "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } case foStdin: { - out << fmt(ANSI_BLUE " at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); + out << fmt(ELLIPSIS_INDENT "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); break; } default: -- cgit v1.2.3 From 402ee8ab64f9b11989cbdcf53f8ca513cb25e23f Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Thu, 28 Apr 2022 13:02:39 +0200 Subject: No point in passing string_views by reference --- src/libutil/error.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 7dc8c1941..3644cada9 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -347,7 +347,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * to make a decision between the two following options. * * ``` long traces - * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, const std::string_view & errorCtx) + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) * { * try { * e->eval(*this, env, v); @@ -361,7 +361,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * ``` * * ``` short traces - * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, const std::string_view & errorCtx) + * inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx) * { * e->eval(*this, env, v); * try { -- cgit v1.2.3 From c94180386182c8cd9e2a4a8053afb5938940c1d2 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Fri, 29 Apr 2022 11:27:38 -0600 Subject: spacing --- src/libutil/error.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 6eb80fb9e..a53e9802e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -99,6 +99,7 @@ struct ErrPos { }; std::optional getCodeLines(const ErrPos & errPos); + void printCodeLines(std::ostream & out, const std::string & prefix, const ErrPos & errPos, @@ -106,7 +107,6 @@ void printCodeLines(std::ostream & out, void printAtPos(const ErrPos & pos, std::ostream & out); - struct Trace { std::optional pos; hintformat hint; -- cgit v1.2.3 From a3c6c5b1c745a72a6a46bdf1a1de7a51a53f76b0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 May 2022 14:37:28 +0200 Subject: nix profile: Support overriding outputs --- src/libutil/experimental-features.cc | 6 ++++-- src/libutil/experimental-features.hh | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index df37edf57..9326cd08c 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -58,11 +58,13 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu return str << showExperimentalFeature(feature); } -void to_json(nlohmann::json& j, const ExperimentalFeature& feature) { +void to_json(nlohmann::json & j, const ExperimentalFeature & feature) +{ j = showExperimentalFeature(feature); } -void from_json(const nlohmann::json& j, ExperimentalFeature& feature) { +void from_json(const nlohmann::json & j, ExperimentalFeature & feature) +{ const std::string input = j; const auto parsed = parseExperimentalFeature(input); diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index a6d080094..57512830c 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -55,7 +55,7 @@ public: * Semi-magic conversion to and from json. * See the nlohmann/json readme for more details. */ -void to_json(nlohmann::json&, const ExperimentalFeature&); -void from_json(const nlohmann::json&, ExperimentalFeature&); +void to_json(nlohmann::json &, const ExperimentalFeature &); +void from_json(const nlohmann::json &, ExperimentalFeature &); } -- cgit v1.2.3 From 1385b2007804c8a0370f2a6555045a00e34b07c7 Mon Sep 17 00:00:00 2001 From: Alain Zscheile Date: Wed, 4 May 2022 07:44:32 +0200 Subject: Get rid of most `.at` calls (#6393) Use one of `get` or `getOr` instead which will either return a null-pointer (with a nicer error message) or a default value when the key is missing. --- src/libutil/experimental-features.cc | 4 +++- src/libutil/tests/tests.cc | 20 ++++++++++++++++++-- src/libutil/util.cc | 14 ++++++++++++++ src/libutil/util.hh | 27 ++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 6 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 9326cd08c..315de64a4 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -35,7 +35,9 @@ const std::optional parseExperimentalFeature(const std::str std::string_view showExperimentalFeature(const ExperimentalFeature feature) { - return stringifiedXpFeatures.at(feature); + const auto ret = get(stringifiedXpFeatures, feature); + assert(ret); + return *ret; } std::set parseFeatures(const std::set & rawFeatures) diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 92972ed14..6e325db98 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -548,7 +548,7 @@ namespace nix { TEST(get, emptyContainer) { StringMap s = { }; - auto expected = std::nullopt; + auto expected = nullptr; ASSERT_EQ(get(s, "one"), expected); } @@ -559,7 +559,23 @@ namespace nix { s["two"] = "er"; auto expected = "yi"; - ASSERT_EQ(get(s, "one"), expected); + ASSERT_EQ(*get(s, "one"), expected); + } + + TEST(getOr, emptyContainer) { + StringMap s = { }; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); + } + + TEST(getOr, getFromContainer) { + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b49c1e466..8bc99f531 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1586,6 +1586,20 @@ std::string stripIndentation(std::string_view s) } +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index a1d0e0e6b..6319ced47 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -543,13 +543,34 @@ std::string stripIndentation(std::string_view s); /* Get a value for the specified key from an associate container. */ template -std::optional get(const T & map, const typename T::key_type & key) +const typename T::mapped_type * get(const T & map, const typename T::key_type & key) { auto i = map.find(key); - if (i == map.end()) return {}; - return std::optional(i->second); + if (i == map.end()) return nullptr; + return &i->second; } +template +typename T::mapped_type * get(T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &i->second; +} + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key); +nlohmann::json * get(nlohmann::json & map, const std::string & key); + +/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ +template +const typename T::mapped_type & getOr(T & map, + const typename T::key_type & key, + const typename T::mapped_type & defaultValue) +{ + auto i = map.find(key); + if (i == map.end()) return defaultValue; + return i->second; +} /* Remove and return the first item from a container. */ template -- cgit v1.2.3 From 3e87c8e62b3eacecde4e9f467b5463fbbdaa6a3f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 11:22:06 +0200 Subject: Move json stuff out of util.cc --- src/libutil/json-utils.hh | 21 +++++++++++++++++++++ src/libutil/util.cc | 15 --------------- src/libutil/util.hh | 3 --- 3 files changed, 21 insertions(+), 18 deletions(-) create mode 100644 src/libutil/json-utils.hh (limited to 'src/libutil') diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh new file mode 100644 index 000000000..b8a031227 --- /dev/null +++ b/src/libutil/json-utils.hh @@ -0,0 +1,21 @@ +#pragma once + +#include + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8bc99f531..d4d78329d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1586,23 +1586,8 @@ std::string stripIndentation(std::string_view s) } -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} - -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} - ////////////////////////////////////////////////////////////////////// - static Sync> windowSize{{0, 0}}; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 6319ced47..09ccfa591 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -558,9 +558,6 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) return &i->second; } -const nlohmann::json * get(const nlohmann::json & map, const std::string & key); -nlohmann::json * get(nlohmann::json & map, const std::string & key); - /* Get a value for the specified key from an associate container, or a default value if the key isn't present. */ template const typename T::mapped_type & getOr(T & map, -- cgit v1.2.3 From e68676e6c859815f40079b6340399d82cc1913b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 4 May 2022 14:32:21 +0200 Subject: Fix the parsing of the sourcehut refs file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since a26be9f3b89be2ee90c6358250b9889b37f95cf8, the same parser is used to parse the result of sourcehut’s `HEAD` endpoint (coming from [git dumb protocol]) and the output of `git ls-remote`. However, they are very slightly different (the former doesn’t specify the current reference since it’s implied to be `HEAD`). Unify both, and make the parser a bit more robust and understandable (by making it more typed and adding tests for it) [git dumb protocol]: https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols#_the_dumb_protocol --- src/libutil/git.cc | 25 +++++++++++++++++++++++++ src/libutil/git.hh | 40 ++++++++++++++++++++++++++++++++++++++++ src/libutil/tests/git.cc | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100644 src/libutil/git.cc create mode 100644 src/libutil/git.hh create mode 100644 src/libutil/tests/git.cc (limited to 'src/libutil') diff --git a/src/libutil/git.cc b/src/libutil/git.cc new file mode 100644 index 000000000..f35c2fdb7 --- /dev/null +++ b/src/libutil/git.cc @@ -0,0 +1,25 @@ +#include "git.hh" + +#include + +namespace nix { +namespace git { + +std::optional parseLsRemoteLine(std::string_view line) +{ + const static std::regex line_regex("^(ref: *)?([^\\s]+)(?:\\t+(.*))?$"); + std::match_results match; + if (!std::regex_match(line.cbegin(), line.cend(), match, line_regex)) + return std::nullopt; + + return LsRemoteRefLine { + .kind = match[1].length() == 0 + ? LsRemoteRefLine::Kind::Object + : LsRemoteRefLine::Kind::Symbolic, + .target = match[2], + .reference = match[3].length() == 0 ? std::nullopt : std::optional{ match[3] } + }; +} + +} +} diff --git a/src/libutil/git.hh b/src/libutil/git.hh new file mode 100644 index 000000000..cb13ef0e5 --- /dev/null +++ b/src/libutil/git.hh @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +namespace nix { + +namespace git { + +// A line from the output of `git ls-remote --symref`. +// +// These can be of two kinds: +// +// - Symbolic references of the form +// +// ref: {target} {reference} +// +// where {target} is itself a reference and {reference} is optional +// +// - Object references of the form +// +// {target} {reference} +// +// where {target} is a commit id and {reference} is mandatory +struct LsRemoteRefLine { + enum struct Kind { + Symbolic, + Object + }; + Kind kind; + std::string target; + std::optional reference; +}; + +std::optional parseLsRemoteLine(std::string_view line); + +} + +} diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc new file mode 100644 index 000000000..5b5715fc2 --- /dev/null +++ b/src/libutil/tests/git.cc @@ -0,0 +1,33 @@ +#include "git.hh" +#include + +namespace nix { + + TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = git::parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); + } + + TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = git::parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); + } + + TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = git::parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); + } +} + -- cgit v1.2.3 From 667074b5867ffe40e3f1c59bd8e4ebf259f86aaa Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Mon, 16 May 2022 09:20:51 -0600 Subject: first whack at passing evalState as an arg to debuggerHook. --- src/libutil/ref.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index f9578afc7..bf26321db 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -7,7 +7,7 @@ namespace nix { /* A simple non-nullable reference-counted pointer. Actually a wrapper - around std::shared_ptr that prevents non-null constructions. */ + around std::shared_ptr that prevents null constructions. */ template class ref { -- cgit v1.2.3 From 5b8c1deb18e0e6fc7a83fb8101cf5fc8dba38843 Mon Sep 17 00:00:00 2001 From: Tony Olagbaiye Date: Fri, 16 Oct 2020 00:35:24 +0100 Subject: fetchTree: Allow fetching plain files Add a new `file` fetcher type, which will fetch a plain file over http(s), or from the local file. Because plain `http(s)://` or `file://` urls can already correspond to `tarball` inputs (if the path ends-up with a know archive extension), the URL parsing logic is a bit convuluted in that: - {http,https,file}:// urls will be interpreted as either a tarball or a file input, depending on the extensions of the path part (so `https://foo.com/bar` will be a `file` input and `https://foo.com/bar.tar.gz` as a `tarball` input) - `file+{something}://` urls will be interpreted as `file` urls (with the `file+` part removed) - `tarball+{something}://` urls will be interpreted as `tarball` urls (with the `tarball+` part removed) Fix #3785 Co-Authored-By: Tony Olagbaiye --- src/libutil/url.cc | 18 ++++++++++++++++++ src/libutil/url.hh | 15 +++++++++++++++ 2 files changed, 33 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f6232d255..5b7abeb49 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -1,6 +1,7 @@ #include "url.hh" #include "url-parts.hh" #include "util.hh" +#include "split.hh" namespace nix { @@ -136,4 +137,21 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +/** + * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' + * into a tuple '(applicationScheme, transportScheme)' + * + * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"} + * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"} + */ +ParsedUrlScheme parseUrlScheme(std::string_view scheme) +{ + auto application = splitPrefixTo(scheme, '+'); + auto transport = scheme; + return ParsedUrlScheme { + .application = application, + .transport = transport, + }; +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 6e77142e3..2a9fb34c1 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -27,4 +27,19 @@ std::map decodeQuery(const std::string & query); ParsedURL parseURL(const std::string & url); +/* + * Although that’s not really standardized anywhere, an number of tools + * use a scheme of the form 'x+y' in urls, where y is the “transport layer” + * scheme, and x is the “application layer” scheme. + * + * For example git uses `git+https` to designate remotes using a Git + * protocol over http. + */ +struct ParsedUrlScheme { + std::optional application; + std::string_view transport; +}; + +ParsedUrlScheme parseUrlScheme(std::string_view scheme); + } -- cgit v1.2.3 From 7a04fb1c56ca60652c2a44019b31fe8cf2e2bc46 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 20 May 2022 08:20:00 -0400 Subject: repl: add repl-flake experimental feature for gating --- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + 2 files changed, 2 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 315de64a4..fa79cca6b 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -13,6 +13,7 @@ std::map stringifiedXpFeatures = { { Xp::RecursiveNix, "recursive-nix" }, { Xp::NoUrlLiterals, "no-url-literals" }, { Xp::FetchClosure, "fetch-closure" }, + { Xp::ReplFlake, "repl-flake" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 57512830c..d09ab025c 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -22,6 +22,7 @@ enum struct ExperimentalFeature RecursiveNix, NoUrlLiterals, FetchClosure, + ReplFlake, }; /** -- cgit v1.2.3 From c156155239dafd68104a843916d8d737e6e61bed Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 26 May 2022 10:53:06 +0200 Subject: createUnixDomainSocket: listen(unix, 5 -> 100) This solves the error error: cannot connect to socket at '/nix/var/nix/daemon-socket/socket': Connection refused on build farm systems that are loaded but operating normally. I've seen this happen on an M1 mac running a loaded hercules-ci-agent. Hercules CI uses multiple worker processes, which may connect to the Nix daemon around the same time. It's not unthinkable that the Nix daemon listening process isn't scheduled until after 6 workers try to connect, especially on a system under load with many workers. Is the increase safe? The number is the number of connections that the kernel will buffer while the listening process hasn't `accept`-ed them yet. It did not - and will not - restrict the total number of daemon forks that a client can create. History The number 5 has remained unchanged since the introduction in nix-worker with 0130ef88ea in 2006. --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d4d78329d..1c19938a8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1818,7 +1818,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) if (chmod(path.c_str(), mode) == -1) throw SysError("changing permissions on '%1%'", path); - if (listen(fdSocket.get(), 5) == -1) + if (listen(fdSocket.get(), 100) == -1) throw SysError("cannot listen on socket '%1%'", path); return fdSocket; -- cgit v1.2.3 From b36d5172cb2cd99f8ae5262b3e3536cceac76b50 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Thu, 26 May 2022 18:36:10 +0100 Subject: src/libutil/json.cc: add missing include for gcc-13 Without the change llvm build fails on this week's gcc-13 snapshot as: src/libutil/json.cc: In function 'void nix::toJSON(std::ostream&, const char*, const char*)': src/libutil/json.cc:33:22: error: 'uint16_t' was not declared in this scope 33 | put(hex[(uint16_t(*i) >> 12) & 0xf]); | ^~~~~~~~ src/libutil/json.cc:5:1: note: 'uint16_t' is defined in header ''; did you forget to '#include '? 4 | #include +++ |+#include 5 | --- src/libutil/json.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libutil') diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 3a981376f..b0a5d7e75 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -1,6 +1,7 @@ #include "json.hh" #include +#include #include namespace nix { -- cgit v1.2.3 From 159b5815b527f466578a2d28fbf832617cc45b88 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 31 Jan 2022 18:03:24 +0100 Subject: repl: `--option pure-eval true` actually enables pure eval mode To quote Eelco in #5867: > Unfortunately we can't do > > evalSettings.pureEval.setDefault(false); > > because then we have to do the same in main.cc (where > pureEval is set to true), and that would allow pure-eval > to be disabled globally from nix.conf. Instead, a command should specify that it should be impure by default. Then, `evalSettings.pureEval` will be set to `false;` unless it's overridden by e.g. a CLI flag. In that case it's IMHO OK to be (theoretically) able to override `pure-eval` via `nix.conf` because it doesn't have an effect on commands where `forceImpureByDefault` returns `false` (i.e. everything where pure eval actually matters). Closes #5867 --- src/libutil/args.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/args.hh b/src/libutil/args.hh index fdd036f9a..07c017719 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -25,6 +25,8 @@ public: /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } + virtual bool forceImpureByDefault() { return false; } + /* Return documentation about this command, in Markdown format. */ virtual std::string doc() { return ""; } -- cgit v1.2.3 From abb80cfa4ca961ee1718480e8252ff76bb2dbb2b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 May 2022 14:28:27 +0200 Subject: Add operator for concatenating strings and string_views --- src/libutil/util.hh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 09ccfa591..16fa6c54c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -700,4 +700,19 @@ template overloaded(Ts...) -> overloaded; std::string showBytes(uint64_t bytes); +/* Provide an addition operator between strings and string_views + inexplicably omitted from the standard library. */ +inline std::string operator + (const std::string & s1, std::string_view s2) +{ + auto s = s1; + s.append(s2); + return s; +} + +inline std::string operator + (std::string && s, std::string_view s2) +{ + s.append(s2); + return s; +} + } -- cgit v1.2.3 From 28e08822a360d396260cec42d92704a3663e6aa9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Jun 2022 16:48:53 +0200 Subject: Avoid unnecessary string copy --- src/libutil/util.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 16fa6c54c..90418b04d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -712,7 +712,7 @@ inline std::string operator + (const std::string & s1, std::string_view s2) inline std::string operator + (std::string && s, std::string_view s2) { s.append(s2); - return s; + return std::move(s); } } -- cgit v1.2.3 From d137ceccefe08250106dcede1f30c270b0f9cf19 Mon Sep 17 00:00:00 2001 From: Fishhh Date: Sun, 5 Jun 2022 18:44:37 +0200 Subject: Fix incorrect comment in `hiliteMatches` --- src/libutil/hilite.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/hilite.cc b/src/libutil/hilite.cc index a5991ca39..e5088230d 100644 --- a/src/libutil/hilite.cc +++ b/src/libutil/hilite.cc @@ -8,9 +8,9 @@ std::string hiliteMatches( std::string_view prefix, std::string_view postfix) { - // Avoid copy on zero matches + // Avoid extra work on zero matches if (matches.size() == 0) - return (std::string) s; + return std::string(s); std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { return a.position() < b.position(); -- cgit v1.2.3 From ca2be509b96a10a2035039a825fc2b292ec0ad4d Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Wed, 15 Jun 2022 16:38:56 -0400 Subject: Verify `$HOME` is owned by current user in `getHome()`, if it exists. Useful because a default `sudo` on darwin doesn't clear `$HOME`, so things like `sudo nix-channel --list` will surprisingly return the USER'S channels, rather than `root`'s. Other counterintuitive outcomes can be seen in this PR description: https://github.com/NixOS/nix/pull/6622 --- src/libutil/util.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1c19938a8..a368ac844 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -574,6 +574,20 @@ Path getHome() static Path homeDir = []() { auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("Couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); + homeDir.reset(); + } + } if (!homeDir) { std::vector buf(16384); struct passwd pwbuf; -- cgit v1.2.3 From f6cf644e5f7da4a0391b10fb31b4b4661c5439dc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 15:35:52 +0200 Subject: Style --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a368ac844..aabd23427 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -580,7 +580,7 @@ Path getHome() int result = stat(homeDir->c_str(), &st); if (result != 0) { if (errno != ENOENT) { - warn("Couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); homeDir.reset(); } } else if (st.st_uid != geteuid()) { -- cgit v1.2.3 From d3176ce076407ef3e63667c0436bccf8be317ae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 22:43:53 +0200 Subject: Fix build-remote in nix-static 'build-remote' is now executed via /proc/self/exe so it always works. --- src/libutil/util.cc | 14 ++++++++++++++ src/libutil/util.hh | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aabd23427..82628461c 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -633,6 +633,20 @@ Path getDataDir() } +std::optional getSelfExe() +{ + static std::optional cached = []() + { + #if __linux__ + return readLink("/proc/self/exe"); + #else + return std::nullopt; + #endif + }(); + return cached; +} + + Paths createDirs(const Path & path) { Paths created; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 90418b04d..d3ed15b0b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -149,10 +149,14 @@ std::vector getConfigDirs(); /* Return $XDG_DATA_HOME or $HOME/.local/share. */ Path getDataDir(); +/* Return the path of the current executable. */ +std::optional getSelfExe(); + /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) { +inline Paths createDirs(PathView path) +{ return createDirs(Path(path)); } -- cgit v1.2.3 From 1e55ee2961eabd6016dfef1793996ded97c9054c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 01:32:17 +0200 Subject: getSelfExe(): Support macOS --- src/libutil/util.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 82628461c..28df30fef 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -29,6 +29,7 @@ #ifdef __APPLE__ #include +#include #endif #ifdef __linux__ @@ -635,10 +636,17 @@ Path getDataDir() std::optional getSelfExe() { - static std::optional cached = []() + static auto cached = []() -> std::optional { #if __linux__ return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; #else return std::nullopt; #endif -- cgit v1.2.3 From 711b2e1f48316d80853635408c518e3562a1fa37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 20 Jun 2022 04:15:38 +0200 Subject: Fix flake input completion for `InstallablesCommand`s Defers completion of flake inputs until the whole command line is parsed so that we know what flakes we need to complete the inputs of. Previously, `nix build flake --update-input ` always behaved like `nix build . --update-input `. --- src/libutil/args.cc | 10 +++++++++- src/libutil/args.hh | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 4b8c55686..44b63f0f6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -124,7 +124,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) bool anyCompleted = false; for (size_t n = 0 ; n < flag.handler.arity; ++n) { if (pos == end) { - if (flag.handler.arity == ArityAny) break; + if (flag.handler.arity == ArityAny || anyCompleted) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } if (auto prefix = needsCompletion(*pos)) { @@ -362,6 +362,14 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } +void MultiCommand::completionHook() +{ + if (command) + return command->second->completionHook(); + else + return Args::completionHook(); +} + nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 07c017719..84866f12b 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -148,6 +148,11 @@ protected: argument (if any) have been processed. */ virtual void initialFlagsProcessed() {} + /* Called after the command line has been processed if we need to generate + completions. Useful for commands that need to know the whole command line + in order to know what completions to generate. */ + virtual void completionHook() { } + public: void addFlag(Flag && flag); @@ -223,6 +228,8 @@ public: bool processArgs(const Strings & args, bool finish) override; + void completionHook() override; + nlohmann::json toJSON() override; }; -- cgit v1.2.3 From a9e75eca00f7efe361545c6e77ecb65244230dc3 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:33 -0700 Subject: error.hh: add additional constructor with explicit errno argument --- src/libutil/error.hh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a53e9802e..3d1479c54 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -204,13 +204,19 @@ public: int errNo; template - SysError(const Args & ... args) + SysError(int errNo_, const Args & ... args) : Error("") { - errNo = errno; + errNo = errNo_; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } + + template + SysError(const Args & ... args) + : SysError(errno, args ...) + { + } }; } -- cgit v1.2.3 From 722de8ddcc875c7e8e9a228f9d88454bae31fd40 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 19 Jul 2022 02:09:46 -0400 Subject: libstore/globals.cc: Move cgroup detection to libutil --- src/libutil/util.cc | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/libutil/util.hh | 3 +++ 2 files changed, 54 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 28df30fef..be6fe091f 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -35,6 +35,9 @@ #ifdef __linux__ #include #include + +#include +#include #endif @@ -788,7 +791,55 @@ void drainFD(int fd, Sink & sink, bool block) } } +////////////////////////////////////////////////////////////////////// + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return 0; + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } + } + } catch (Error &) { ignoreException(); } + #endif + + return 0; +} ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index d3ed15b0b..29227ecc6 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -182,6 +182,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); +/* If cgroups are active, attempt to calculate the number of CPUs available. + If cgroups are unavailable or if cpu.max is set to "max", return 0. */ +unsigned int getMaxCPU(); /* Automatic cleanup of resources. */ -- cgit v1.2.3 From 8119390abcbb25e849acc50d0af0b37d85adfb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:08 +0100 Subject: Move some fs-related functions to their own file Unclutter `util.cc` a bit --- src/libutil/filesystem.cc | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/libutil/util.cc | 38 -------------------------------------- 2 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 src/libutil/filesystem.cc (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc new file mode 100644 index 000000000..e67383e6f --- /dev/null +++ b/src/libutil/filesystem.cc @@ -0,0 +1,44 @@ +#include + +#include "util.hh" +#include "types.hh" + +namespace nix { + +void createSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } +} + +void replaceSymlink(const Path & target, const Path & link, + std::optional mtime) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp, mtime); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmp, link); + + break; + } +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..3f3695f0d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -681,44 +681,6 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } -} - - -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp, mtime); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); - - break; - } -} - - void readFull(int fd, char * buf, size_t count) { while (count) { -- cgit v1.2.3 From c2de0a232c1cfddb1f1385ffd23dd43a2099458e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:46 +0100 Subject: =?UTF-8?q?Create=20a=20wrapper=20around=20stdlib=E2=80=99s=20`ren?= =?UTF-8?q?ame`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Directly takes some c++ strings, and gently throws an exception on error (rather than having to inline this logic everywhere) --- src/libutil/filesystem.cc | 9 +++++++-- src/libutil/util.hh | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index e67383e6f..33a8d81a6 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -34,11 +34,16 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - if (rename(tmp.c_str(), link.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmp, link); + moveFile(tmp, link); break; } } +void moveFile(const Path & oldName, const Path & newName) +{ + if (::rename(oldName.c_str(), newName.c_str())) + throw SysError("renaming '%1%' to '%2%'", oldName, newName); +} + } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29227ecc6..564d36e79 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,6 +168,8 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ -- cgit v1.2.3 From 6f89fb60088c4bc1513a005f0350c2bc13068892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 16:13:29 +0100 Subject: rename: Fallback to a copy if the filesystems mismatch In `nix::rename`, if the call to `rename` fails with `EXDEV` (failure because the source and the destination are in a different filesystems) switch to copying and removing the source. To avoid having to re-implement the copy manually, I switched the function to use the c++17 `filesystem` library (which has a `copy` function that should do what we want). Fix #6262 --- src/libutil/filesystem.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 33a8d81a6..cf99b848a 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,8 +1,11 @@ #include +#include #include "util.hh" #include "types.hh" +namespace fs = std::filesystem; + namespace nix { void createSymlink(const Path & target, const Path & link, @@ -42,8 +45,16 @@ void replaceSymlink(const Path & target, const Path & link, void moveFile(const Path & oldName, const Path & newName) { - if (::rename(oldName.c_str(), newName.c_str())) - throw SysError("renaming '%1%' to '%2%'", oldName, newName); + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + try { + fs::rename(oldPath, newPath); + } catch (fs::filesystem_error & e) { + if (e.code().value() == EXDEV) { + fs::copy(oldName, newName, fs::copy_options::copy_symlinks); + fs::remove_all(oldName); + } + } } } -- cgit v1.2.3 From c5db1821a94406eaff86341220bc301ba1dac82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 18 Mar 2022 14:25:56 +0100 Subject: Re-implement the recursive directory copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recursive copy from the stl doesn’t exactly do what we need because 1. It doesn’t delete things as we go 2. It doesn’t keep the mtime, which change the nars So re-implement it ourselves. A bit dull, but that way we have what we want --- src/libutil/filesystem.cc | 52 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index cf99b848a..198db2832 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -43,6 +43,53 @@ void replaceSymlink(const Path & target, const Path & link, } } +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + void moveFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); @@ -51,8 +98,9 @@ void moveFile(const Path & oldName, const Path & newName) fs::rename(oldPath, newPath); } catch (fs::filesystem_error & e) { if (e.code().value() == EXDEV) { - fs::copy(oldName, newName, fs::copy_options::copy_symlinks); - fs::remove_all(oldName); + fs::remove(newPath); + warn("Copying %s to %s", oldName, newName); + copy(fs::directory_entry(oldPath), newPath, true); } } } -- cgit v1.2.3 From a4f0fd633c9ec865d385b34bdc51923c204e7ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 11:52:39 +0200 Subject: Link against c++fs on darwin Required by the old clang version --- src/libutil/local.mk | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/local.mk b/src/libutil/local.mk index f880c0fc5..13e8d426a 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,3 +11,7 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif + +ifdef HOST_DARWIN + libutil_LDFLAGS += -lc++fs +endif -- cgit v1.2.3 From d71d9e9fbfc972514b0b940181ab7f9e766f41f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:10:36 +0200 Subject: moveFile -> renameFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `move` tends to have this `mv` connotation of “I will copy it for you if needs be” --- src/libutil/filesystem.cc | 4 ++-- src/libutil/util.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 198db2832..fee40d09e 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -37,7 +37,7 @@ void replaceSymlink(const Path & target, const Path & link, throw; } - moveFile(tmp, link); + renameFile(tmp, link); break; } @@ -90,7 +90,7 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) } } -void moveFile(const Path & oldName, const Path & newName) +void renameFile(const Path & oldName, const Path & newName) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 564d36e79..186513cea 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -168,7 +168,7 @@ void createSymlink(const Path & target, const Path & link, void replaceSymlink(const Path & target, const Path & link, std::optional mtime = {}); -void moveFile(const Path & src, const Path & dst); +void renameFile(const Path & src, const Path & dst); /* Wrappers arount read()/write() that read/write exactly the -- cgit v1.2.3 From 90f968073338d6ba276994bc281fa5efa5306e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:19:42 +0200 Subject: Only use `renameFile` where needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In most places the fallback to copying isn’t needed and can actually be bad, so we’d rather not transparently fallback --- src/libutil/filesystem.cc | 14 +++++++++----- src/libutil/util.hh | 9 +++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index fee40d09e..9cc18507b 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -54,7 +54,6 @@ void setWriteTime(const fs::path & p, const struct stat & st) .tv_sec = st.st_mtime, .tv_usec = 0, }; - warn("Setting the mtime of %s to %d", p.c_str(), st.st_mtim.tv_sec); if (lutimes(p.c_str(), times) != 0) throw SysError("changing modification time of '%s'", p); } @@ -92,14 +91,19 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) void renameFile(const Path & oldName, const Path & newName) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ try { - fs::rename(oldPath, newPath); + renameFile(oldName, newName); } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); if (e.code().value() == EXDEV) { fs::remove(newPath); - warn("Copying %s to %s", oldName, newName); + warn("Can’t rename %s as %s, copying instead", oldName, newName); copy(fs::directory_entry(oldPath), newPath, true); } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 186513cea..cd83f250f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -170,6 +170,15 @@ void replaceSymlink(const Path & target, const Path & link, void renameFile(const Path & src, const Path & dst); +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + /* Wrappers arount read()/write() that read/write exactly the requested number of bytes. */ -- cgit v1.2.3 From 1ba5b3e001c3da3c2e2a4dc7d475da1b20f53638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:48:31 +0200 Subject: Make `moveFile` more atomic Rather than directly copying the source to its dest, copy it first to a temporary location, and eventually move that temporary. That way, the move is at least atomic from the point-of-view of the destination --- src/libutil/filesystem.cc | 62 ++++++++++++++++++++++++++++++++++++++++++++++- src/libutil/util.cc | 55 ----------------------------------------- 2 files changed, 61 insertions(+), 56 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 9cc18507b..403389e60 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,6 +1,7 @@ #include #include +#include "finally.hh" #include "util.hh" #include "types.hh" @@ -8,6 +9,59 @@ namespace fs = std::filesystem; namespace nix { +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + void createSymlink(const Path & target, const Path & link, std::optional mtime) { @@ -101,10 +155,16 @@ void moveFile(const Path & oldName, const Path & newName) } catch (fs::filesystem_error & e) { auto oldPath = fs::path(oldName); auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; if (e.code().value() == EXDEV) { fs::remove(newPath); warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), newPath, true); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); } } } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3f3695f0d..2a1bb770a 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -508,61 +508,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed) } -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); - else - return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); -} - - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - - std::string getUserName() { auto pw = getpwuid(geteuid()); -- cgit v1.2.3 From ccbd906c86cb10e5c7b49753e057fec4ab46233c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Aug 2022 17:45:11 +0200 Subject: Fix NIX_COUNT_CALLS=1 Also, make the JSON writer support std::string_view. Fixes #6857. --- src/libutil/json.cc | 21 +++++++++------------ src/libutil/json.hh | 11 +++++------ src/libutil/tests/json.cc | 4 ++-- 3 files changed, 16 insertions(+), 20 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/json.cc b/src/libutil/json.cc index b0a5d7e75..abe0e6e74 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -6,7 +6,8 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end) +template<> +void toJSON(std::ostream & str, const std::string_view & s) { constexpr size_t BUF_SIZE = 4096; char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts @@ -21,7 +22,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) }; put('"'); - for (auto i = start; i != end; i++) { + for (auto i = s.begin(); i != s.end(); i++) { if (bufPos >= BUF_SIZE) flush(); if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } else if (*i == '\n') { put('\\'); put('n'); } @@ -44,7 +45,7 @@ void toJSON(std::ostream & str, const char * start, const char * end) void toJSON(std::ostream & str, const char * s) { - if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); + if (!s) str << "null"; else toJSON(str, std::string_view(s)); } template<> void toJSON(std::ostream & str, const int & n) { str << n; } @@ -55,11 +56,7 @@ template<> void toJSON(std::ostream & str, const long long & n) { str template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } template<> void toJSON(std::ostream & str, const float & n) { str << n; } template<> void toJSON(std::ostream & str, const double & n) { str << n; } - -template<> void toJSON(std::ostream & str, const std::string & s) -{ - toJSON(str, s.c_str(), s.c_str() + s.size()); -} +template<> void toJSON(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); } template<> void toJSON(std::ostream & str, const bool & b) { @@ -154,7 +151,7 @@ JSONObject::~JSONObject() } } -void JSONObject::attr(const std::string & s) +void JSONObject::attr(std::string_view s) { comma(); toJSON(state->str, s); @@ -162,19 +159,19 @@ void JSONObject::attr(const std::string & s) if (state->indent) state->str << ' '; } -JSONList JSONObject::list(const std::string & name) +JSONList JSONObject::list(std::string_view name) { attr(name); return JSONList(state); } -JSONObject JSONObject::object(const std::string & name) +JSONObject JSONObject::object(std::string_view name) { attr(name); return JSONObject(state); } -JSONPlaceholder JSONObject::placeholder(const std::string & name) +JSONPlaceholder JSONObject::placeholder(std::string_view name) { attr(name); return JSONPlaceholder(state); diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 83213ca66..3790b1a2e 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -6,7 +6,6 @@ namespace nix { -void toJSON(std::ostream & str, const char * start, const char * end); void toJSON(std::ostream & str, const char * s); template @@ -107,7 +106,7 @@ private: open(); } - void attr(const std::string & s); + void attr(std::string_view s); public: @@ -128,18 +127,18 @@ public: ~JSONObject(); template - JSONObject & attr(const std::string & name, const T & v) + JSONObject & attr(std::string_view name, const T & v) { attr(name); toJSON(state->str, v); return *this; } - JSONList list(const std::string & name); + JSONList list(std::string_view name); - JSONObject object(const std::string & name); + JSONObject object(std::string_view name); - JSONPlaceholder placeholder(const std::string & name); + JSONPlaceholder placeholder(std::string_view name); }; class JSONPlaceholder : JSONWriter diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc index dea73f53a..156286999 100644 --- a/src/libutil/tests/json.cc +++ b/src/libutil/tests/json.cc @@ -102,8 +102,8 @@ namespace nix { TEST(toJSON, substringEscape) { std::stringstream out; - const char *s = "foo\t"; - toJSON(out, s+3, s + strlen(s)); + std::string_view s = "foo\t"; + toJSON(out, s.substr(3)); ASSERT_EQ(out.str(), "\"\\t\""); } -- cgit v1.2.3 From cb6794a0d983eb364601f26fc32ead98ed67bfb4 Mon Sep 17 00:00:00 2001 From: Dave Nicponski Date: Sun, 7 Aug 2022 10:13:11 -0400 Subject: Do not spam logs if the owned-homedir check results in a noop --- src/libutil/util.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index be6fe091f..e11cb9c60 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -577,6 +577,7 @@ Path getHome() { static Path homeDir = []() { + std::optional unownedUserHomeDir = {}; auto homeDir = getEnv("HOME"); if (homeDir) { // Only use $HOME if doesn't exist or is owned by the current user. @@ -588,8 +589,7 @@ Path getHome() homeDir.reset(); } } else if (st.st_uid != geteuid()) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file", *homeDir); - homeDir.reset(); + unownedUserHomeDir.swap(homeDir); } } if (!homeDir) { @@ -600,6 +600,9 @@ Path getHome() || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } } return *homeDir; }(); -- cgit v1.2.3 From f4a8426098481245e6fb1388de48366c5c82991f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 8 Aug 2022 14:34:22 +0200 Subject: Remove the explicit `c++fs` linkage on darwin Doesn't seem needed on a recent-enough clang anymore (and even seems to break stuff) --- src/libutil/local.mk | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 13e8d426a..f880c0fc5 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -11,7 +11,3 @@ libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif - -ifdef HOST_DARWIN - libutil_LDFLAGS += -lc++fs -endif -- cgit v1.2.3 From 53e7b7e8ac44704199a868c0f6850ac53a0b8ad1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 12:28:02 +0200 Subject: Remove warnLargeDump() This message was unhelpful (#1184) and probably misleading since memory is O(1) in most cases now. --- src/libutil/serialise.cc | 20 -------------------- src/libutil/serialise.hh | 4 +--- 2 files changed, 1 insertion(+), 23 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 8ff904583..2c3597775 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -48,24 +48,9 @@ FdSink::~FdSink() } -size_t threshold = 256 * 1024 * 1024; - -static void warnLargeDump() -{ - warn("dumping very large path (> 256 MiB); this may run out of memory"); -} - - void FdSink::write(std::string_view data) { written += data.size(); - static bool warned = false; - if (warn && !warned) { - if (written > threshold) { - warnLargeDump(); - warned = true; - } - } try { writeFull(fd, data); } catch (SysError & e) { @@ -448,11 +433,6 @@ Error readError(Source & source) void StringSink::operator () (std::string_view data) { - static bool warned = false; - if (!warned && s.size() > threshold) { - warnLargeDump(); - warned = true; - } s.append(data); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 13da26c6a..84847835a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -97,19 +97,17 @@ protected: struct FdSink : BufferedSink { int fd; - bool warn = false; size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } FdSink(FdSink&&) = default; - FdSink& operator=(FdSink && s) + FdSink & operator=(FdSink && s) { flush(); fd = s.fd; s.fd = -1; - warn = s.warn; written = s.written; return *this; } -- cgit v1.2.3 From 8188b1d0abc2eba6497b5dc47f7e848cbacb7677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 19 Aug 2022 00:59:04 +0200 Subject: json: write null on abnormal placeholder destruction Avoids leaving dangling attributes like { "foo": } in case of exceptions. --- src/libutil/json.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/json.cc b/src/libutil/json.cc index abe0e6e74..2f9e97ff5 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -193,7 +193,11 @@ JSONObject JSONPlaceholder::object() JSONPlaceholder::~JSONPlaceholder() { - assert(!first || std::uncaught_exceptions()); + if (first) { + assert(std::uncaught_exceptions()); + if (state->stack != 0) + write(nullptr); + } } } -- cgit v1.2.3 From bb411e4ae16d6a5c61ea595c0c12e2ecee081ff9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 22:36:40 +0200 Subject: Fix progress bar flicker with -L This was caused by -L calling setLogFormat() again, which caused the creation of a new progress bar without destroying the old one. So we had two progress bars clobbering each other. We should change 'logger' to be a smart pointer, but I'll do that in a future PR. Fixes #6931. --- src/libutil/logging.hh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 6f81b92de..d0817b4a9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -111,6 +111,9 @@ public: virtual std::optional ask(std::string_view s) { return {}; } + + virtual void setPrintBuildLogs(bool printBuildLogs) + { } }; ActivityId getCurActivity(); -- cgit v1.2.3 From d365cced4fadbbc63f0c39902a7091e1a34c34de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 13 Sep 2022 16:58:32 +0200 Subject: Trim option descriptions This removes unintended blank lines in Markdown when the description is a multiline string literal. --- src/libutil/args.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 44b63f0f6..753980fd4 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -216,7 +216,7 @@ nlohmann::json Args::toJSON() if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); if (flag->description != "") - j["description"] = flag->description; + j["description"] = trim(flag->description); j["category"] = flag->category; if (flag->handler.arity != ArityAny) j["arity"] = flag->handler.arity; @@ -237,7 +237,7 @@ nlohmann::json Args::toJSON() } auto res = nlohmann::json::object(); - res["description"] = description(); + res["description"] = trim(description()); res["flags"] = std::move(flags); res["args"] = std::move(args); auto s = doc(); @@ -379,7 +379,7 @@ nlohmann::json MultiCommand::toJSON() auto j = command->toJSON(); auto cat = nlohmann::json::object(); cat["id"] = command->category(); - cat["description"] = categories[command->category()]; + cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); cmds[name] = std::move(j); } -- cgit v1.2.3 From 1b595026e18afb050de3f62ded8f7180bc8b2b0e Mon Sep 17 00:00:00 2001 From: squalus Date: Mon, 19 Sep 2022 11:15:31 -0700 Subject: Improve durability of schema version file writes - call close explicitly in writeFile to prevent the close exception from being ignored - fsync after writing schema file to flush data to disk - fsync schema file parent to flush metadata to disk https://github.com/NixOS/nix/issues/7064 --- src/libutil/util.cc | 38 ++++++++++++++++++++++++++++++++++++-- src/libutil/util.hh | 8 ++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 96ac11ea2..623b74bdd 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -353,7 +353,7 @@ void readFile(const Path & path, Sink & sink) } -void writeFile(const Path & path, std::string_view s, mode_t mode) +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -364,10 +364,16 @@ void writeFile(const Path & path, std::string_view s, mode_t mode) e.addTrace({}, "writing file '%1%'", path); throw; } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); } -void writeFile(const Path & path, Source & source, mode_t mode) +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); if (!fd) @@ -386,6 +392,20 @@ void writeFile(const Path & path, Source & source, mode_t mode) e.addTrace({}, "writing file '%1%'", path); throw; } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); } std::string readLine(int fd) @@ -841,6 +861,20 @@ void AutoCloseFD::close() } } +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + AutoCloseFD::operator bool() const { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index cd83f250f..e5c678682 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -115,9 +115,12 @@ std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666); +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); -void writeFile(const Path & path, Source & source, mode_t mode = 0666); +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/* Flush a file's parent directory to disk */ +void syncParent(const Path & path); /* Read a line from a file descriptor. */ std::string readLine(int fd); @@ -231,6 +234,7 @@ public: explicit operator bool() const; int release(); void close(); + void fsync(); }; -- cgit v1.2.3 From 223f8dace091a0549f86b966b1d935c42bb01efd Mon Sep 17 00:00:00 2001 From: squalus Date: Thu, 22 Sep 2022 12:47:43 -0700 Subject: archive: check close errors when extracting nars --- src/libutil/archive.cc | 7 +++++++ src/libutil/archive.hh | 1 + 2 files changed, 8 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 30b471af5..4b0636129 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -234,6 +234,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) else if (s == "contents" && type == tpRegular) { parseContents(sink, source, path); + sink.closeRegularFile(); } else if (s == "executable" && type == tpRegular) { @@ -324,6 +325,12 @@ struct RestoreSink : ParseSink if (!fd) throw SysError("creating file '%1%'", p); } + void closeRegularFile() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + void isExecutable() override { struct stat st; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 79ce08df0..ac4183bf5 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -60,6 +60,7 @@ struct ParseSink virtual void createDirectory(const Path & path) { }; virtual void createRegularFile(const Path & path) { }; + virtual void closeRegularFile() { }; virtual void isExecutable() { }; virtual void preallocateContents(uint64_t size) { }; virtual void receiveContents(std::string_view data) { }; -- cgit v1.2.3 From b945b844a9ce8479872f6280aedde27e2974b7f3 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 17 Oct 2022 03:05:02 +0200 Subject: Initial frames support --- src/libutil/error.cc | 9 +++++++-- src/libutil/error.hh | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 5f86e1e76..cf4f4a56f 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -9,9 +9,9 @@ namespace nix { const std::string nativeSystem = SYSTEM; -void BaseError::addTrace(std::optional e, hintformat hint) +void BaseError::addTrace(std::optional e, hintformat hint, bool frame) { - err.traces.push_front(Trace { .pos = e, .hint = hint }); + err.traces.push_front(Trace { .pos = e, .hint = hint, .frame = frame }); } // c++ std::exception descendants must have a 'const char* what()' function. @@ -382,6 +382,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * */ + bool frameOnly = false; if (!einfo.traces.empty()) { unsigned int count = 0; for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { @@ -391,7 +392,11 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s } if (iter->hint.str().empty()) continue; + if (frameOnly && !iter->frame) continue; + count++; + frameOnly = iter->frame; + oss << "\n" << "… " << iter->hint.str() << "\n"; if (iter->pos.has_value() && (*iter->pos)) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 50335676e..bf99581e2 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -110,6 +110,7 @@ void printAtPos(const ErrPos & pos, std::ostream & out); struct Trace { std::optional pos; hintformat hint; + bool frame; }; struct ErrorInfo { @@ -188,7 +189,7 @@ public: addTrace(e, hintfmt(std::string(fs), args...)); } - void addTrace(std::optional e, hintformat hint); + void addTrace(std::optional e, hintformat hint, bool frame = false); bool hasTrace() const { return !err.traces.empty(); } }; -- cgit v1.2.3 From e93bf69b448d4f4ce6c3fe7b7acfa904afe058c0 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Tue, 25 Oct 2022 01:46:10 +0200 Subject: Rework error throwing, and test it --- src/libutil/error.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index bf99581e2..6db77bcbf 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -192,6 +192,8 @@ public: void addTrace(std::optional e, hintformat hint, bool frame = false); bool hasTrace() const { return !err.traces.empty(); } + + const ErrorInfo & info() { return err; }; }; #define MakeError(newClass, superClass) \ -- cgit v1.2.3 From 34ea0e2e7b72aa70b4b562eef77c7f3617fed1bb Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Tue, 1 Nov 2022 15:46:30 +0100 Subject: tarfile: set directory mode to at least 0500, don't extract fflags We don't need SGID, or any ACL's. We also want to keep every dir +rx. --- src/libutil/tarfile.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index a7db58559..238d0a7a6 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -77,9 +77,7 @@ TarArchive::~TarArchive() static void extract_archive(TarArchive & archive, const Path & destDir) { - int flags = ARCHIVE_EXTRACT_FFLAGS - | ARCHIVE_EXTRACT_PERM - | ARCHIVE_EXTRACT_TIME + int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -98,6 +96,10 @@ static void extract_archive(TarArchive & archive, const Path & destDir) archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + // sources can and do contain dirs with no rx bits + if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) + archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); + // Patch hardlink path const char *original_hardlink = archive_entry_hardlink(entry); if (original_hardlink) { -- cgit v1.2.3 From 40911d7dec75541a400fe8f556d4c70a7f845fac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Nov 2022 13:30:35 +0100 Subject: Remove stray tab --- src/libutil/experimental-features.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 30d071408..670079019 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -14,8 +14,8 @@ std::map stringifiedXpFeatures = { { Xp::NoUrlLiterals, "no-url-literals" }, { Xp::FetchClosure, "fetch-closure" }, { Xp::ReplFlake, "repl-flake" }, - { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::SystemdCgroup, "systemd-cgroup" }, + { Xp::AutoAllocateUids, "auto-allocate-uids" }, + { Xp::SystemdCgroup, "systemd-cgroup" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) -- cgit v1.2.3 From 6c6eff8ac40e2f5d7b6ff8e772feebb1aa484039 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Nov 2022 17:24:12 +0100 Subject: Remove the SystemdCgroup feature --- src/libutil/experimental-features.cc | 1 - src/libutil/experimental-features.hh | 1 - 2 files changed, 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 670079019..0f05f3752 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -15,7 +15,6 @@ std::map stringifiedXpFeatures = { { Xp::FetchClosure, "fetch-closure" }, { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, - { Xp::SystemdCgroup, "systemd-cgroup" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index c749d4767..cf0c06eac 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -24,7 +24,6 @@ enum struct ExperimentalFeature FetchClosure, ReplFlake, AutoAllocateUids, - SystemdCgroup, }; /** -- cgit v1.2.3 From 09f00dd4d01aa1b6866978d162022133e521614f Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 16 Nov 2022 16:49:49 +0100 Subject: Replace src/libutil/json.cc with nlohmann json generation --- src/libutil/json.cc | 203 ---------------------------------------------- src/libutil/json.hh | 185 ------------------------------------------ src/libutil/tests/json.cc | 193 ------------------------------------------- 3 files changed, 581 deletions(-) delete mode 100644 src/libutil/json.cc delete mode 100644 src/libutil/json.hh delete mode 100644 src/libutil/tests/json.cc (limited to 'src/libutil') diff --git a/src/libutil/json.cc b/src/libutil/json.cc deleted file mode 100644 index 2f9e97ff5..000000000 --- a/src/libutil/json.cc +++ /dev/null @@ -1,203 +0,0 @@ -#include "json.hh" - -#include -#include -#include - -namespace nix { - -template<> -void toJSON(std::ostream & str, const std::string_view & s) -{ - constexpr size_t BUF_SIZE = 4096; - char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts - size_t bufPos = 0; - - const auto flush = [&] { - str.write(buf, bufPos); - bufPos = 0; - }; - const auto put = [&] (char c) { - buf[bufPos++] = c; - }; - - put('"'); - for (auto i = s.begin(); i != s.end(); i++) { - if (bufPos >= BUF_SIZE) flush(); - if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } - else if (*i == '\n') { put('\\'); put('n'); } - else if (*i == '\r') { put('\\'); put('r'); } - else if (*i == '\t') { put('\\'); put('t'); } - else if (*i >= 0 && *i < 32) { - const char hex[17] = "0123456789abcdef"; - put('\\'); - put('u'); - put(hex[(uint16_t(*i) >> 12) & 0xf]); - put(hex[(uint16_t(*i) >> 8) & 0xf]); - put(hex[(uint16_t(*i) >> 4) & 0xf]); - put(hex[(uint16_t(*i) >> 0) & 0xf]); - } - else put(*i); - } - put('"'); - flush(); -} - -void toJSON(std::ostream & str, const char * s) -{ - if (!s) str << "null"; else toJSON(str, std::string_view(s)); -} - -template<> void toJSON(std::ostream & str, const int & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned int & n) { str << n; } -template<> void toJSON(std::ostream & str, const long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } -template<> void toJSON(std::ostream & str, const long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } -template<> void toJSON(std::ostream & str, const float & n) { str << n; } -template<> void toJSON(std::ostream & str, const double & n) { str << n; } -template<> void toJSON(std::ostream & str, const std::string & s) { toJSON(str, (std::string_view) s); } - -template<> void toJSON(std::ostream & str, const bool & b) -{ - str << (b ? "true" : "false"); -} - -template<> void toJSON(std::ostream & str, const std::nullptr_t & b) -{ - str << "null"; -} - -JSONWriter::JSONWriter(std::ostream & str, bool indent) - : state(new JSONState(str, indent)) -{ - state->stack++; -} - -JSONWriter::JSONWriter(JSONState * state) - : state(state) -{ - state->stack++; -} - -JSONWriter::~JSONWriter() -{ - if (state) { - assertActive(); - state->stack--; - if (state->stack == 0) delete state; - } -} - -void JSONWriter::comma() -{ - assertActive(); - if (first) { - first = false; - } else { - state->str << ','; - } - if (state->indent) indent(); -} - -void JSONWriter::indent() -{ - state->str << '\n' << std::string(state->depth * 2, ' '); -} - -void JSONList::open() -{ - state->depth++; - state->str << '['; -} - -JSONList::~JSONList() -{ - state->depth--; - if (state->indent && !first) indent(); - state->str << "]"; -} - -JSONList JSONList::list() -{ - comma(); - return JSONList(state); -} - -JSONObject JSONList::object() -{ - comma(); - return JSONObject(state); -} - -JSONPlaceholder JSONList::placeholder() -{ - comma(); - return JSONPlaceholder(state); -} - -void JSONObject::open() -{ - state->depth++; - state->str << '{'; -} - -JSONObject::~JSONObject() -{ - if (state) { - state->depth--; - if (state->indent && !first) indent(); - state->str << "}"; - } -} - -void JSONObject::attr(std::string_view s) -{ - comma(); - toJSON(state->str, s); - state->str << ':'; - if (state->indent) state->str << ' '; -} - -JSONList JSONObject::list(std::string_view name) -{ - attr(name); - return JSONList(state); -} - -JSONObject JSONObject::object(std::string_view name) -{ - attr(name); - return JSONObject(state); -} - -JSONPlaceholder JSONObject::placeholder(std::string_view name) -{ - attr(name); - return JSONPlaceholder(state); -} - -JSONList JSONPlaceholder::list() -{ - assertValid(); - first = false; - return JSONList(state); -} - -JSONObject JSONPlaceholder::object() -{ - assertValid(); - first = false; - return JSONObject(state); -} - -JSONPlaceholder::~JSONPlaceholder() -{ - if (first) { - assert(std::uncaught_exceptions()); - if (state->stack != 0) - write(nullptr); - } -} - -} diff --git a/src/libutil/json.hh b/src/libutil/json.hh deleted file mode 100644 index 3790b1a2e..000000000 --- a/src/libutil/json.hh +++ /dev/null @@ -1,185 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace nix { - -void toJSON(std::ostream & str, const char * s); - -template -void toJSON(std::ostream & str, const T & n); - -class JSONWriter -{ -protected: - - struct JSONState - { - std::ostream & str; - bool indent; - size_t depth = 0; - size_t stack = 0; - JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } - ~JSONState() - { - assert(stack == 0); - } - }; - - JSONState * state; - - bool first = true; - - JSONWriter(std::ostream & str, bool indent); - - JSONWriter(JSONState * state); - - ~JSONWriter(); - - void assertActive() - { - assert(state->stack != 0); - } - - void comma(); - - void indent(); -}; - -class JSONObject; -class JSONPlaceholder; - -class JSONList : JSONWriter -{ -private: - - friend class JSONObject; - friend class JSONPlaceholder; - - void open(); - - JSONList(JSONState * state) - : JSONWriter(state) - { - open(); - } - -public: - - JSONList(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } - - ~JSONList(); - - template - JSONList & elem(const T & v) - { - comma(); - toJSON(state->str, v); - return *this; - } - - JSONList list(); - - JSONObject object(); - - JSONPlaceholder placeholder(); -}; - -class JSONObject : JSONWriter -{ -private: - - friend class JSONList; - friend class JSONPlaceholder; - - void open(); - - JSONObject(JSONState * state) - : JSONWriter(state) - { - open(); - } - - void attr(std::string_view s); - -public: - - JSONObject(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - open(); - } - - JSONObject(const JSONObject & obj) = delete; - - JSONObject(JSONObject && obj) - : JSONWriter(obj.state) - { - obj.state = 0; - } - - ~JSONObject(); - - template - JSONObject & attr(std::string_view name, const T & v) - { - attr(name); - toJSON(state->str, v); - return *this; - } - - JSONList list(std::string_view name); - - JSONObject object(std::string_view name); - - JSONPlaceholder placeholder(std::string_view name); -}; - -class JSONPlaceholder : JSONWriter -{ - -private: - - friend class JSONList; - friend class JSONObject; - - JSONPlaceholder(JSONState * state) - : JSONWriter(state) - { - } - - void assertValid() - { - assertActive(); - assert(first); - } - -public: - - JSONPlaceholder(std::ostream & str, bool indent = false) - : JSONWriter(str, indent) - { - } - - ~JSONPlaceholder(); - - template - void write(const T & v) - { - assertValid(); - first = false; - toJSON(state->str, v); - } - - JSONList list(); - - JSONObject object(); -}; - -} diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc deleted file mode 100644 index 156286999..000000000 --- a/src/libutil/tests/json.cc +++ /dev/null @@ -1,193 +0,0 @@ -#include "json.hh" -#include -#include - -namespace nix { - - /* ---------------------------------------------------------------------------- - * toJSON - * --------------------------------------------------------------------------*/ - - TEST(toJSON, quotesCharPtr) { - const char* input = "test"; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "\"test\""); - } - - TEST(toJSON, quotesStdString) { - std::string input = "test"; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "\"test\""); - } - - TEST(toJSON, convertsNullptrtoNull) { - auto input = nullptr; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "null"); - } - - TEST(toJSON, convertsNullToNull) { - const char* input = 0; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "null"); - } - - - TEST(toJSON, convertsFloat) { - auto input = 1.024f; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "1.024"); - } - - TEST(toJSON, convertsDouble) { - const double input = 1.024; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "1.024"); - } - - TEST(toJSON, convertsBool) { - auto input = false; - std::stringstream out; - toJSON(out, input); - - ASSERT_EQ(out.str(), "false"); - } - - TEST(toJSON, quotesTab) { - std::stringstream out; - toJSON(out, "\t"); - - ASSERT_EQ(out.str(), "\"\\t\""); - } - - TEST(toJSON, quotesNewline) { - std::stringstream out; - toJSON(out, "\n"); - - ASSERT_EQ(out.str(), "\"\\n\""); - } - - TEST(toJSON, quotesCreturn) { - std::stringstream out; - toJSON(out, "\r"); - - ASSERT_EQ(out.str(), "\"\\r\""); - } - - TEST(toJSON, quotesCreturnNewLine) { - std::stringstream out; - toJSON(out, "\r\n"); - - ASSERT_EQ(out.str(), "\"\\r\\n\""); - } - - TEST(toJSON, quotesDoublequotes) { - std::stringstream out; - toJSON(out, "\""); - - ASSERT_EQ(out.str(), "\"\\\"\""); - } - - TEST(toJSON, substringEscape) { - std::stringstream out; - std::string_view s = "foo\t"; - toJSON(out, s.substr(3)); - - ASSERT_EQ(out.str(), "\"\\t\""); - } - - /* ---------------------------------------------------------------------------- - * JSONObject - * --------------------------------------------------------------------------*/ - - TEST(JSONObject, emptyObject) { - std::stringstream out; - { - JSONObject t(out); - } - ASSERT_EQ(out.str(), "{}"); - } - - TEST(JSONObject, objectWithList) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.list("list"); - l.elem("element"); - } - ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); - } - - TEST(JSONObject, objectWithListIndent) { - std::stringstream out; - { - JSONObject t(out, true); - auto l = t.list("list"); - l.elem("element"); - } - ASSERT_EQ(out.str(), -R"#({ - "list": [ - "element" - ] -})#"); - } - - TEST(JSONObject, objectWithPlaceholderAndList) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.placeholder("list"); - l.list().elem("element"); - } - - ASSERT_EQ(out.str(), R"#({"list":["element"]})#"); - } - - TEST(JSONObject, objectWithPlaceholderAndObject) { - std::stringstream out; - { - JSONObject t(out); - auto l = t.placeholder("object"); - l.object().attr("key", "value"); - } - - ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#"); - } - - /* ---------------------------------------------------------------------------- - * JSONList - * --------------------------------------------------------------------------*/ - - TEST(JSONList, empty) { - std::stringstream out; - { - JSONList l(out); - } - ASSERT_EQ(out.str(), R"#([])#"); - } - - TEST(JSONList, withElements) { - std::stringstream out; - { - JSONList l(out); - l.elem("one"); - l.object(); - l.placeholder().write("three"); - } - ASSERT_EQ(out.str(), R"#(["one",{},"three"])#"); - } -} - -- cgit v1.2.3 From f1ab082ac4f589a36a9eb0cd98d1cc235eedc419 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 09:37:11 +0100 Subject: createTempDir(): Use std::atomic --- src/libutil/filesystem.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 403389e60..3a732cff8 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -1,5 +1,6 @@ #include #include +#include #include "finally.hh" #include "util.hh" @@ -10,7 +11,7 @@ namespace fs = std::filesystem; namespace nix { static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - int & counter) + std::atomic & counter) { tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); if (includePid) @@ -22,9 +23,9 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, Path createTempDir(const Path & tmpRoot, const Path & prefix, bool includePid, bool useGlobalCounter, mode_t mode) { - static int globalCounter = 0; - int localCounter = 0; - int & counter(useGlobalCounter ? globalCounter : localCounter); + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); while (1) { checkInterrupt(); -- cgit v1.2.3 From 128910ba23f586ba1765a137ecff23cfd22cff89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 10:39:28 +0100 Subject: Separate cgroup support from auto-uid-allocation The new experimental feature 'cgroups' enables the use of cgroups for all builds. This allows better containment and enables setting resource limits and getting some build stats. --- src/libutil/experimental-features.cc | 1 + src/libutil/experimental-features.hh | 1 + 2 files changed, 2 insertions(+) (limited to 'src/libutil') diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 0f05f3752..e0902971e 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -15,6 +15,7 @@ std::map stringifiedXpFeatures = { { Xp::FetchClosure, "fetch-closure" }, { Xp::ReplFlake, "repl-flake" }, { Xp::AutoAllocateUids, "auto-allocate-uids" }, + { Xp::Cgroups, "cgroups" }, }; const std::optional parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index cf0c06eac..af775feb0 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -24,6 +24,7 @@ enum struct ExperimentalFeature FetchClosure, ReplFlake, AutoAllocateUids, + Cgroups, }; /** -- cgit v1.2.3 From 1211e59a038379026496bbee4b203bbd66833b01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 12:38:03 +0100 Subject: Move cgroup.{cc,hh} to libutil --- src/libutil/cgroup.cc | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/libutil/cgroup.hh | 27 +++++++++++ 2 files changed, 158 insertions(+) create mode 100644 src/libutil/cgroup.cc create mode 100644 src/libutil/cgroup.hh (limited to 'src/libutil') diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc new file mode 100644 index 000000000..f693d77be --- /dev/null +++ b/src/libutil/cgroup.cc @@ -0,0 +1,131 @@ +#if __linux__ + +#include "cgroup.hh" +#include "util.hh" + +#include +#include +#include +#include +#include + +#include + +namespace nix { + +// FIXME: obsolete, check for cgroup2 +std::map getCgroups(const Path & cgroupFile) +{ + std::map cgroups; + + for (auto & line : tokenizeString>(readFile(cgroupFile), "\n")) { + static std::regex regex("([0-9]+):([^:]*):(.*)"); + std::smatch match; + if (!std::regex_match(line, match, regex)) + throw Error("invalid line '%s' in '%s'", line, cgroupFile); + + std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2]; + cgroups.insert_or_assign(name, match[3]); + } + + return cgroups; +} + +static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) +{ + if (!pathExists(cgroup)) return {}; + + auto procsFile = cgroup + "/cgroup.procs"; + + if (!pathExists(procsFile)) + throw Error("'%s' is not a cgroup", cgroup); + + /* Use the fast way to kill every process in a cgroup, if + available. */ + auto killFile = cgroup + "/cgroup.kill"; + if (pathExists(killFile)) + writeFile(killFile, "1"); + + /* Otherwise, manually kill every process in the subcgroups and + this cgroup. */ + for (auto & entry : readDirectory(cgroup)) { + if (entry.type != DT_DIR) continue; + destroyCgroup(cgroup + "/" + entry.name, false); + } + + int round = 1; + + std::unordered_set pidsShown; + + while (true) { + auto pids = tokenizeString>(readFile(procsFile)); + + if (pids.empty()) break; + + if (round > 20) + throw Error("cannot kill cgroup '%s'", cgroup); + + for (auto & pid_s : pids) { + pid_t pid; + if (auto o = string2Int(pid_s)) + pid = *o; + else + throw Error("invalid pid '%s'", pid); + if (pidsShown.insert(pid).second) { + try { + auto cmdline = readFile(fmt("/proc/%d/cmdline", pid)); + using namespace std::string_literals; + warn("killing stray builder process %d (%s)...", + pid, trim(replaceStrings(cmdline, "\0"s, " "))); + } catch (SysError &) { + } + } + // FIXME: pid wraparound + if (kill(pid, SIGKILL) == -1 && errno != ESRCH) + throw SysError("killing member %d of cgroup '%s'", pid, cgroup); + } + + auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); + if (sleep.count() > 100) + printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + std::this_thread::sleep_for(sleep); + round++; + } + + CgroupStats stats; + + if (returnStats) { + auto cpustatPath = cgroup + "/cpu.stat"; + + if (pathExists(cpustatPath)) { + for (auto & line : tokenizeString>(readFile(cpustatPath), "\n")) { + std::string_view userPrefix = "user_usec "; + if (hasPrefix(line, userPrefix)) { + auto n = string2Int(line.substr(userPrefix.size())); + if (n) stats.cpuUser = std::chrono::microseconds(*n); + } + + std::string_view systemPrefix = "system_usec "; + if (hasPrefix(line, systemPrefix)) { + auto n = string2Int(line.substr(systemPrefix.size())); + if (n) stats.cpuSystem = std::chrono::microseconds(*n); + } + } + } + + } + + if (rmdir(cgroup.c_str()) == -1) + throw SysError("deleting cgroup '%s'", cgroup); + + return stats; +} + +CgroupStats destroyCgroup(const Path & cgroup) +{ + return destroyCgroup(cgroup, true); +} + +} + +#endif diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh new file mode 100644 index 000000000..3ead4735f --- /dev/null +++ b/src/libutil/cgroup.hh @@ -0,0 +1,27 @@ +#pragma once + +#if __linux__ + +#include +#include + +#include "types.hh" + +namespace nix { + +std::map getCgroups(const Path & cgroupFile); + +struct CgroupStats +{ + std::optional cpuUser, cpuSystem; +}; + +/* Destroy the cgroup denoted by 'path'. The postcondition is that + 'path' does not exist, and thus any processes in the cgroup have + been killed. Also return statistics from the cgroup just before + destruction. */ +CgroupStats destroyCgroup(const Path & cgroup); + +} + +#endif -- cgit v1.2.3 From 1e6a5d1ff6e8ef5bf340502f74c4d5039cedc67a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 12:57:41 +0100 Subject: Clean up cgroup handling in getMaxCPU() Also, don't assume in LocalDerivationGoal that cgroups are mounted on /sys/fs/cgroup. --- src/libutil/cgroup.cc | 17 +++++++++++++++++ src/libutil/cgroup.hh | 2 ++ src/libutil/util.cc | 49 +++++++++++++++---------------------------------- 3 files changed, 34 insertions(+), 34 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index f693d77be..a008481ca 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "finally.hh" #include #include @@ -10,9 +11,25 @@ #include #include +#include namespace nix { +std::optional getCgroupFS() +{ + static auto res = [&]() -> std::optional { + auto fp = fopen("/proc/mounts", "r"); + if (!fp) return std::nullopt; + Finally delFP = [&]() { fclose(fp); }; + while (auto ent = getmntent(fp)) + if (std::string_view(ent->mnt_type) == "cgroup2") + return ent->mnt_dir; + + return std::nullopt; + }(); + return res; +} + // FIXME: obsolete, check for cgroup2 std::map getCgroups(const Path & cgroupFile) { diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh index 3ead4735f..d08c8ad29 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/cgroup.hh @@ -9,6 +9,8 @@ namespace nix { +std::optional getCgroupFS(); + std::map getCgroups(const Path & cgroupFile); struct CgroupStats diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 623b74bdd..2c2aae82e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -2,6 +2,7 @@ #include "sync.hh" #include "finally.hh" #include "serialise.hh" +#include "cgroup.hh" #include #include @@ -36,7 +37,6 @@ #include #include -#include #include #endif @@ -727,43 +727,24 @@ unsigned int getMaxCPU() { #if __linux__ try { - FILE *fp = fopen("/proc/mounts", "r"); - if (!fp) - return 0; + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; - Strings cgPathParts; + if (!pathExists("/proc/self/cgroup")) return 0; - struct mntent *ent; - while ((ent = getmntent(fp))) { - std::string mountType, mountPath; + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; - mountType = ent->mnt_type; - mountPath = ent->mnt_dir; + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - if (mountType == "cgroup2") { - cgPathParts.push_back(mountPath); - break; - } - } - - fclose(fp); - - if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { - std::string currentCgroup = readFile("/proc/self/cgroup"); - Strings cgValues = tokenizeString(currentCgroup, ":"); - cgPathParts.push_back(trim(cgValues.back(), "\n")); - cgPathParts.push_back("cpu.max"); - std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); - - if (pathExists(fullCgPath)) { - std::string cpuMax = readFile(fullCgPath); - std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); - std::string quota = cpuMaxParts[0]; - std::string period = trim(cpuMaxParts[1], "\n"); - - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } + if (pathExists(cpuFile)) { + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); } } catch (Error &) { ignoreException(); } #endif -- cgit v1.2.3 From fa99ef6a879e77024d60e73901a4773c6756c1bb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 15:03:40 +0100 Subject: getMaxCPU(): Lower verbosity level for ignored exceptions Fixes #7268. --- src/libutil/util.cc | 22 +++++++++------------- src/libutil/util.hh | 2 +- 2 files changed, 10 insertions(+), 14 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 2c2aae82e..a93ef1901 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -730,23 +730,19 @@ unsigned int getMaxCPU() auto cgroupFS = getCgroupFS(); if (!cgroupFS) return 0; - if (!pathExists("/proc/self/cgroup")) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroups = getCgroups("/proc/self/cgroupp"); auto cgroup = cgroups[""]; if (cgroup == "") return 0; auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - if (pathExists(cpuFile)) { - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") return std::ceil(std::stoi(quota) / std::stof(period)); - } - } catch (Error &) { ignoreException(); } + } catch (Error &) { ignoreException(lvlDebug); } #endif return 0; @@ -1408,7 +1404,7 @@ std::string shellEscape(const std::string_view s) } -void ignoreException() +void ignoreException(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -1416,7 +1412,7 @@ void ignoreException() try { throw; } catch (std::exception & e) { - printError("error (ignored): %1%", e.what()); + printMsg(lvl, "error (ignored): %1%", e.what()); } } catch (...) { } } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index e5c678682..94d8cc555 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -528,7 +528,7 @@ std::string shellEscape(const std::string_view s); /* Exception handling in destructors: print an error message, then ignore the exception. */ -void ignoreException(); +void ignoreException(Verbosity lvl = lvlError); -- cgit v1.2.3 From cccd57c022753c3ad727847c9c83c9d2c9c639e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 4 Dec 2022 18:22:11 +0100 Subject: getMaxCPU: fix cgroup path Given this typo I am not sure if it has been tested. --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a93ef1901..4f2caaa40 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -730,7 +730,7 @@ unsigned int getMaxCPU() auto cgroupFS = getCgroupFS(); if (!cgroupFS) return 0; - auto cgroups = getCgroups("/proc/self/cgroupp"); + auto cgroups = getCgroups("/proc/self/cgroup"); auto cgroup = cgroups[""]; if (cgroup == "") return 0; -- cgit v1.2.3 From 703d863a48f549b2626382eda407ffae779f8725 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Dec 2022 12:58:58 +0100 Subject: Trivial changes from the lazy-trees branch --- src/libutil/archive.cc | 4 ---- src/libutil/archive.hh | 4 +++- src/libutil/fmt.hh | 2 +- src/libutil/logging.cc | 12 ++---------- src/libutil/logging.hh | 8 ++++++-- src/libutil/ref.hh | 5 +++++ src/libutil/serialise.cc | 2 +- src/libutil/serialise.hh | 14 +++----------- src/libutil/util.cc | 15 +++++++++++++++ src/libutil/util.hh | 24 ++++++++++++++++++++++++ 10 files changed, 60 insertions(+), 30 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4b0636129..0e2b9d12c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -35,10 +35,6 @@ static ArchiveSettings archiveSettings; static GlobalConfig::Register rArchiveSettings(&archiveSettings); -const std::string narVersionMagic1 = "nix-archive-1"; - -static std::string caseHackSuffix = "~nix~case~hack~"; - PathFilter defaultPathFilter = [](const Path &) { return true; }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index ac4183bf5..e42dea540 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -103,7 +103,9 @@ void copyNAR(Source & source, Sink & sink); void copyPath(const Path & from, const Path & to); -extern const std::string narVersionMagic1; +inline constexpr std::string_view narVersionMagic1 = "nix-archive-1"; + +inline constexpr std::string_view caseHackSuffix = "~nix~case~hack~"; } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 7664e5c04..e879fd3b8 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -148,7 +148,7 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args) return f; } -inline hintformat hintfmt(std::string plain_string) +inline hintformat hintfmt(const std::string & plain_string) { // we won't be receiving any args in this case, so just print the original string return hintfmt("%s", normaltxt(plain_string)); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index cb2b15b41..ac86d8ac2 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -105,14 +105,6 @@ public: Verbosity verbosity = lvlInfo; -void warnOnce(bool & haveWarned, const FormatOrString & fs) -{ - if (!haveWarned) { - warn(fs.s); - haveWarned = true; - } -} - void writeToStderr(std::string_view s) { try { @@ -130,11 +122,11 @@ Logger * makeSimpleLogger(bool printBuildLogs) return new SimpleLogger(printBuildLogs); } -std::atomic nextId{(uint64_t) getpid() << 32}; +std::atomic nextId{0}; Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s, const Logger::Fields & fields, ActivityId parent) - : logger(logger), id(nextId++) + : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32)) { logger.startActivity(id, lvl, type, s, fields, parent); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index d0817b4a9..4642c49f7 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -82,7 +82,7 @@ public: log(lvlInfo, fs); } - virtual void logEI(const ErrorInfo &ei) = 0; + virtual void logEI(const ErrorInfo & ei) = 0; void logEI(Verbosity lvl, ErrorInfo ei) { @@ -225,7 +225,11 @@ inline void warn(const std::string & fs, const Args & ... args) logger->warn(f.str()); } -void warnOnce(bool & haveWarned, const FormatOrString & fs); +#define warnOnce(haveWarned, args...) \ + if (!haveWarned) { \ + haveWarned = true; \ + warn(args); \ + } void writeToStderr(std::string_view s); diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index bf26321db..7d38b059c 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -83,6 +83,11 @@ public: return p != other.p; } + bool operator < (const ref & other) const + { + return p < other.p; + } + private: template diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 2c3597775..c653db9d0 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -338,7 +338,7 @@ Sink & operator << (Sink & sink, const StringSet & s) Sink & operator << (Sink & sink, const Error & ex) { - auto info = ex.info(); + auto & info = ex.info(); sink << "Error" << info.level diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 84847835a..7da5b07fd 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -331,17 +331,9 @@ T readNum(Source & source) unsigned char buf[8]; source((char *) buf, sizeof(buf)); - uint64_t n = - ((uint64_t) buf[0]) | - ((uint64_t) buf[1] << 8) | - ((uint64_t) buf[2] << 16) | - ((uint64_t) buf[3] << 24) | - ((uint64_t) buf[4] << 32) | - ((uint64_t) buf[5] << 40) | - ((uint64_t) buf[6] << 48) | - ((uint64_t) buf[7] << 56); - - if (n > (uint64_t)std::numeric_limits::max()) + auto n = readLittleEndian(buf); + + if (n > (uint64_t) std::numeric_limits::max()) throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); return (T) n; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 4f2caaa40..993dc1cb6 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1594,6 +1594,21 @@ std::string stripIndentation(std::string_view s) } +std::pair getLine(std::string_view s) +{ + auto newline = s.find('\n'); + + if (newline == s.npos) { + return {s, ""}; + } else { + auto line = s.substr(0, newline); + if (!line.empty() && line[line.size() - 1] == '\r') + line = line.substr(0, line.size() - 1); + return {line, s.substr(newline + 1)}; + } +} + + ////////////////////////////////////////////////////////////////////// static Sync> windowSize{{0, 0}}; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 94d8cc555..3caa95fca 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -510,6 +510,17 @@ std::optional string2Float(const std::string_view s) } +/* Convert a little-endian integer to host order. */ +template +T readLittleEndian(unsigned char * p) +{ + T x = 0; + for (size_t i = 0; i < sizeof(x); ++i) + x |= ((T) *p++) << (i * 8); + return x; +} + + /* Return true iff `s' starts with `prefix'. */ bool hasPrefix(std::string_view s, std::string_view prefix); @@ -563,6 +574,12 @@ std::string base64Decode(std::string_view s); std::string stripIndentation(std::string_view s); +/* Get the prefix of 's' up to and excluding the next line break (LF + optionally preceded by CR), and the remainder following the line + break. */ +std::pair getLine(std::string_view s); + + /* Get a value for the specified key from an associate container. */ template const typename T::mapped_type * get(const T & map, const typename T::key_type & key) @@ -737,4 +754,11 @@ inline std::string operator + (std::string && s, std::string_view s2) return std::move(s); } +inline std::string operator + (std::string_view s1, const char * s2) +{ + std::string s(s1); + s.append(s2); + return s; +} + } -- cgit v1.2.3 From 8272cd9deca99b84fe18cddd561f24bb69249b57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Dec 2022 12:36:19 +0100 Subject: Optimize string concatenation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libutil/util.hh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 3caa95fca..2f869d909 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -756,7 +756,9 @@ inline std::string operator + (std::string && s, std::string_view s2) inline std::string operator + (std::string_view s1, const char * s2) { - std::string s(s1); + std::string s; + s.reserve(s1.size() + strlen(s2)); + s.append(s1); s.append(s2); return s; } -- cgit v1.2.3 From 786402365e9c819235636d9300bd25c362a29db7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Dec 2022 12:40:51 +0100 Subject: Cleanup --- src/libutil/util.hh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 2f869d909..9b149de80 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -515,8 +515,9 @@ template T readLittleEndian(unsigned char * p) { T x = 0; - for (size_t i = 0; i < sizeof(x); ++i) - x |= ((T) *p++) << (i * 8); + for (size_t i = 0; i < sizeof(x); ++i, ++p) { + x |= ((T) *p) << (i * 8); + } return x; } -- cgit v1.2.3 From 173dcb0af9249487c2d9ad5de7218fcf203873bd Mon Sep 17 00:00:00 2001 From: Florian Friesdorf Date: Tue, 22 Nov 2022 12:46:55 +0000 Subject: Don't reverse stack trace when showing When debugging nix expressions the outermost trace tends to be more useful than the innermost. It is therefore printed last to save developers from scrolling. --- src/libutil/error.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 9172f67a6..9cac6ac91 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -287,7 +287,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s // traces if (showTrace && !einfo.traces.empty()) { - for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { + for (auto iter = einfo.traces.begin(); iter != einfo.traces.end(); ++iter) { oss << "\n" << "… " << iter->hint.str() << "\n"; if (iter->pos.has_value() && (*iter->pos)) { -- cgit v1.2.3 From d269976be6def2928e6a315ab2b85b947f4308f2 Mon Sep 17 00:00:00 2001 From: Florian Friesdorf Date: Tue, 22 Nov 2022 16:45:58 +0000 Subject: Show stack trace above error message Save developers from scrolling by displaying the error message last, below the stack trace. --- src/libutil/error.cc | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 9cac6ac91..449baaad1 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -262,6 +262,28 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s prefix += ":" ANSI_NORMAL " "; std::ostringstream oss; + + // traces + if (showTrace && !einfo.traces.empty()) { + for (auto iter = einfo.traces.begin(); iter != einfo.traces.end(); ++iter) { + oss << "\n" << "… " << iter->hint.str() << "\n"; + + if (iter->pos.has_value() && (*iter->pos)) { + auto pos = iter->pos.value(); + oss << "\n"; + printAtPos(pos, oss); + + auto loc = getCodeLines(pos); + if (loc.has_value()) { + oss << "\n"; + printCodeLines(oss, "", pos, *loc); + oss << "\n"; + } + } + } + oss << "\n" << prefix; + } + oss << einfo.msg << "\n"; if (einfo.errPos.has_value() && *einfo.errPos) { @@ -285,26 +307,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s "?" << std::endl; } - // traces - if (showTrace && !einfo.traces.empty()) { - for (auto iter = einfo.traces.begin(); iter != einfo.traces.end(); ++iter) { - oss << "\n" << "… " << iter->hint.str() << "\n"; - - if (iter->pos.has_value() && (*iter->pos)) { - auto pos = iter->pos.value(); - oss << "\n"; - printAtPos(pos, oss); - - auto loc = getCodeLines(pos); - if (loc.has_value()) { - oss << "\n"; - printCodeLines(oss, "", pos, *loc); - oss << "\n"; - } - } - } - } - out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str())); return out; -- cgit v1.2.3 From 8618c6cc75e19ed658649bd806b218de954ea3bc Mon Sep 17 00:00:00 2001 From: Florian Friesdorf Date: Fri, 9 Dec 2022 17:36:25 +0000 Subject: Simplify loop, feedback from @tfc and @Ericson2314 --- src/libutil/error.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 449baaad1..3bb3efb0e 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -265,11 +265,11 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s // traces if (showTrace && !einfo.traces.empty()) { - for (auto iter = einfo.traces.begin(); iter != einfo.traces.end(); ++iter) { - oss << "\n" << "… " << iter->hint.str() << "\n"; + for (const auto & trace : einfo.traces) { + oss << "\n" << "… " << trace.hint.str() << "\n"; - if (iter->pos.has_value() && (*iter->pos)) { - auto pos = iter->pos.value(); + if (trace.pos.has_value() && (*trace.pos)) { + auto pos = trace.pos.value(); oss << "\n"; printAtPos(pos, oss); -- cgit v1.2.3 From 900b85408435f6f7118e5d46f606d377452cdb12 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Dec 2022 19:57:32 +0100 Subject: Add CanonPath wrapper to represent canonicalized paths --- src/libutil/canon-path.cc | 103 ++++++++++++++++++++++++ src/libutil/canon-path.hh | 173 ++++++++++++++++++++++++++++++++++++++++ src/libutil/tests/canon-path.cc | 155 +++++++++++++++++++++++++++++++++++ 3 files changed, 431 insertions(+) create mode 100644 src/libutil/canon-path.cc create mode 100644 src/libutil/canon-path.hh create mode 100644 src/libutil/tests/canon-path.cc (limited to 'src/libutil') diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc new file mode 100644 index 000000000..b132b4262 --- /dev/null +++ b/src/libutil/canon-path.cc @@ -0,0 +1,103 @@ +#include "canon-path.hh" +#include "util.hh" + +namespace nix { + +CanonPath CanonPath::root = CanonPath("/"); + +CanonPath::CanonPath(std::string_view raw) + : path(absPath((Path) raw, "/")) +{ } + +CanonPath::CanonPath(std::string_view raw, const CanonPath & root) + : path(absPath((Path) raw, root.abs())) +{ } + +std::optional CanonPath::parent() const +{ + if (isRoot()) return std::nullopt; + return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/')))); +} + +void CanonPath::pop() +{ + assert(!isRoot()); + path.resize(std::max((size_t) 1, path.rfind('/'))); +} + +bool CanonPath::isWithin(const CanonPath & parent) const +{ + return !( + path.size() < parent.path.size() + || path.substr(0, parent.path.size()) != parent.path + || (parent.path.size() > 1 && path.size() > parent.path.size() + && path[parent.path.size()] != '/')); +} + +CanonPath CanonPath::removePrefix(const CanonPath & prefix) const +{ + assert(isWithin(prefix)); + if (prefix.isRoot()) return *this; + if (path.size() == prefix.path.size()) return root; + return CanonPath(unchecked_t(), path.substr(prefix.path.size())); +} + +void CanonPath::extend(const CanonPath & x) +{ + if (x.isRoot()) return; + if (isRoot()) + path += x.rel(); + else + path += x.abs(); +} + +CanonPath CanonPath::operator + (const CanonPath & x) const +{ + auto res = *this; + res.extend(x); + return res; +} + +void CanonPath::push(std::string_view c) +{ + assert(c.find('/') == c.npos); + assert(c != "." && c != ".."); + if (!isRoot()) path += '/'; + path += c; +} + +CanonPath CanonPath::operator + (std::string_view c) const +{ + auto res = *this; + res.push(c); + return res; +} + +bool CanonPath::isAllowed(const std::set & allowed) const +{ + /* Check if `this` is an exact match or the parent of an + allowed path. */ + auto lb = allowed.lower_bound(*this); + if (lb != allowed.end()) { + if (lb->isWithin(*this)) + return true; + } + + /* Check if a parent of `this` is allowed. */ + auto path = *this; + while (!path.isRoot()) { + path.pop(); + if (allowed.count(path)) + return true; + } + + return false; +} + +std::ostream & operator << (std::ostream & stream, const CanonPath & path) +{ + stream << path.abs(); + return stream; +} + +} diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh new file mode 100644 index 000000000..c5e7f0596 --- /dev/null +++ b/src/libutil/canon-path.hh @@ -0,0 +1,173 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nix { + +/* A canonical representation of a path. It ensures the following: + + - It always starts with a slash. + + - It never ends with a slash, except if the path is "/". + + - A slash is never followed by a slash (i.e. no empty components). + + - There are no components equal to '.' or '..'. + + Note that the path does not need to correspond to an actually + existing path, and there is no guarantee that symlinks are + resolved. +*/ +class CanonPath +{ + std::string path; + +public: + + /* Construct a canon path from a non-canonical path. Any '.', '..' + or empty components are removed. */ + CanonPath(std::string_view raw); + + explicit CanonPath(const char * raw) + : CanonPath(std::string_view(raw)) + { } + + struct unchecked_t { }; + + CanonPath(unchecked_t _, std::string path) + : path(std::move(path)) + { } + + static CanonPath root; + + /* If `raw` starts with a slash, return + `CanonPath(raw)`. Otherwise return a `CanonPath` representing + `root + "/" + raw`. */ + CanonPath(std::string_view raw, const CanonPath & root); + + bool isRoot() const + { return path.size() <= 1; } + + explicit operator std::string_view() const + { return path; } + + const std::string & abs() const + { return path; } + + /* Like abs(), but return an empty string if this path is + '/'. Thus the returned string never ends in a slash. */ + const std::string & absOrEmpty() const + { + const static std::string epsilon; + return isRoot() ? epsilon : path; + } + + const char * c_str() const + { return path.c_str(); } + + std::string_view rel() const + { return ((std::string_view) path).substr(1); } + + struct Iterator + { + std::string_view remaining; + size_t slash; + + Iterator(std::string_view remaining) + : remaining(remaining) + , slash(remaining.find('/')) + { } + + bool operator != (const Iterator & x) const + { return remaining.data() != x.remaining.data(); } + + const std::string_view operator * () const + { return remaining.substr(0, slash); } + + void operator ++ () + { + if (slash == remaining.npos) + remaining = remaining.substr(remaining.size()); + else { + remaining = remaining.substr(slash + 1); + slash = remaining.find('/'); + } + } + }; + + Iterator begin() const { return Iterator(rel()); } + Iterator end() const { return Iterator(rel().substr(path.size() - 1)); } + + std::optional parent() const; + + /* Remove the last component. Panics if this path is the root. */ + void pop(); + + std::optional dirOf() const + { + if (isRoot()) return std::nullopt; + return path.substr(0, path.rfind('/')); + } + + std::optional baseName() const + { + if (isRoot()) return std::nullopt; + return ((std::string_view) path).substr(path.rfind('/') + 1); + } + + bool operator == (const CanonPath & x) const + { return path == x.path; } + + bool operator != (const CanonPath & x) const + { return path != x.path; } + + /* Compare paths lexicographically except that path separators + are sorted before any other character. That is, in the sorted order + a directory is always followed directly by its children. For + instance, 'foo' < 'foo/bar' < 'foo!'. */ + bool operator < (const CanonPath & x) const + { + auto i = path.begin(); + auto j = x.path.begin(); + for ( ; i != path.end() && j != x.path.end(); ++i, ++j) { + auto c_i = *i; + if (c_i == '/') c_i = 0; + auto c_j = *j; + if (c_j == '/') c_j = 0; + if (c_i < c_j) return true; + if (c_i > c_j) return false; + } + return i == path.end() && j != x.path.end(); + } + + /* Return true if `this` is equal to `parent` or a child of + `parent`. */ + bool isWithin(const CanonPath & parent) const; + + CanonPath removePrefix(const CanonPath & prefix) const; + + /* Append another path to this one. */ + void extend(const CanonPath & x); + + /* Concatenate two paths. */ + CanonPath operator + (const CanonPath & x) const; + + /* Add a path component to this one. It must not contain any slashes. */ + void push(std::string_view c); + + CanonPath operator + (std::string_view c) const; + + /* Check whether access to this path is allowed, which is the case + if 1) `this` is within any of the `allowed` paths; or 2) any of + the `allowed` paths are within `this`. (The latter condition + ensures access to the parents of allowed paths.) */ + bool isAllowed(const std::set & allowed) const; +}; + +std::ostream & operator << (std::ostream & stream, const CanonPath & path); + +} diff --git a/src/libutil/tests/canon-path.cc b/src/libutil/tests/canon-path.cc new file mode 100644 index 000000000..c1c5adadf --- /dev/null +++ b/src/libutil/tests/canon-path.cc @@ -0,0 +1,155 @@ +#include "canon-path.hh" + +#include + +namespace nix { + + TEST(CanonPath, basic) { + { + CanonPath p("/"); + ASSERT_EQ(p.abs(), "/"); + ASSERT_EQ(p.rel(), ""); + ASSERT_EQ(p.baseName(), std::nullopt); + ASSERT_EQ(p.dirOf(), std::nullopt); + ASSERT_FALSE(p.parent()); + } + + { + CanonPath p("/foo//"); + ASSERT_EQ(p.abs(), "/foo"); + ASSERT_EQ(p.rel(), "foo"); + ASSERT_EQ(*p.baseName(), "foo"); + ASSERT_EQ(*p.dirOf(), ""); // FIXME: do we want this? + ASSERT_EQ(p.parent()->abs(), "/"); + } + + { + CanonPath p("foo/bar"); + ASSERT_EQ(p.abs(), "/foo/bar"); + ASSERT_EQ(p.rel(), "foo/bar"); + ASSERT_EQ(*p.baseName(), "bar"); + ASSERT_EQ(*p.dirOf(), "/foo"); + ASSERT_EQ(p.parent()->abs(), "/foo"); + } + + { + CanonPath p("foo//bar/"); + ASSERT_EQ(p.abs(), "/foo/bar"); + ASSERT_EQ(p.rel(), "foo/bar"); + ASSERT_EQ(*p.baseName(), "bar"); + ASSERT_EQ(*p.dirOf(), "/foo"); + } + } + + TEST(CanonPath, pop) { + CanonPath p("foo/bar/x"); + ASSERT_EQ(p.abs(), "/foo/bar/x"); + p.pop(); + ASSERT_EQ(p.abs(), "/foo/bar"); + p.pop(); + ASSERT_EQ(p.abs(), "/foo"); + p.pop(); + ASSERT_EQ(p.abs(), "/"); + } + + TEST(CanonPath, removePrefix) { + CanonPath p1("foo/bar"); + CanonPath p2("foo/bar/a/b/c"); + ASSERT_EQ(p2.removePrefix(p1).abs(), "/a/b/c"); + ASSERT_EQ(p1.removePrefix(p1).abs(), "/"); + ASSERT_EQ(p1.removePrefix(CanonPath("/")).abs(), "/foo/bar"); + } + + TEST(CanonPath, iter) { + { + CanonPath p("a//foo/bar//"); + std::vector ss; + for (auto & c : p) ss.push_back(c); + ASSERT_EQ(ss, std::vector({"a", "foo", "bar"})); + } + + { + CanonPath p("/"); + std::vector ss; + for (auto & c : p) ss.push_back(c); + ASSERT_EQ(ss, std::vector()); + } + } + + TEST(CanonPath, concat) { + { + CanonPath p1("a//foo/bar//"); + CanonPath p2("xyzzy/bla"); + ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla"); + } + + { + CanonPath p1("/"); + CanonPath p2("/a/b"); + ASSERT_EQ((p1 + p2).abs(), "/a/b"); + } + + { + CanonPath p1("/a/b"); + CanonPath p2("/"); + ASSERT_EQ((p1 + p2).abs(), "/a/b"); + } + + { + CanonPath p("/foo/bar"); + ASSERT_EQ((p + "x").abs(), "/foo/bar/x"); + } + + { + CanonPath p("/"); + ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar"); + } + } + + TEST(CanonPath, within) { + { + ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo"))); + ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo"))); + ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar"))); + ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/"))); + ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/"))); + } + } + + TEST(CanonPath, sort) { + ASSERT_FALSE(CanonPath("foo") < CanonPath("foo")); + ASSERT_TRUE (CanonPath("foo") < CanonPath("foo/bar")); + ASSERT_TRUE (CanonPath("foo/bar") < CanonPath("foo!")); + ASSERT_FALSE(CanonPath("foo!") < CanonPath("foo")); + ASSERT_TRUE (CanonPath("foo") < CanonPath("foo!")); + } + + TEST(CanonPath, allowed) { + { + std::set allowed { + CanonPath("foo/bar"), + CanonPath("foo!"), + CanonPath("xyzzy"), + CanonPath("a/b/c"), + }; + + ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("foo").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed)); + ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed)); + ASSERT_TRUE (CanonPath("/").isAllowed(allowed)); + } + } +} -- cgit v1.2.3 From b3fdab28a216683365f7f04bfa9bbc5cd122d753 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 13 Dec 2022 00:48:04 +0100 Subject: Introduce AbstractPos This makes the position object used in exceptions abstract, with a method getSource() to get the source code of the file in which the error originated. This is needed for lazy trees because source files don't necessarily exist in the filesystem, and we don't want to make libutil depend on the InputAccessor type in libfetcher. --- src/libutil/error.cc | 144 +++++++++++++------------------------------------ src/libutil/error.hh | 58 ++++++-------------- src/libutil/logging.cc | 32 +++++------ 3 files changed, 72 insertions(+), 162 deletions(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 3bb3efb0e..1a1aecea5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -9,9 +9,9 @@ namespace nix { const std::string nativeSystem = SYSTEM; -void BaseError::addTrace(std::optional e, hintformat hint) +void BaseError::addTrace(std::shared_ptr && e, hintformat hint) { - err.traces.push_front(Trace { .pos = e, .hint = hint }); + err.traces.push_front(Trace { .pos = std::move(e), .hint = hint }); } // c++ std::exception descendants must have a 'const char* what()' function. @@ -30,91 +30,46 @@ const std::string & BaseError::calcWhat() const std::optional ErrorInfo::programName = std::nullopt; -std::ostream & operator<<(std::ostream & os, const hintformat & hf) +std::ostream & operator <<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -std::string showErrPos(const ErrPos & errPos) +std::ostream & operator <<(std::ostream & str, const AbstractPos & pos) { - if (errPos.line > 0) { - if (errPos.column > 0) { - return fmt("%d:%d", errPos.line, errPos.column); - } else { - return fmt("%d", errPos.line); - } - } - else { - return ""; - } + pos.print(str); + str << ":" << pos.line; + if (pos.column > 0) + str << ":" << pos.column; + return str; } -std::optional getCodeLines(const ErrPos & errPos) +std::optional AbstractPos::getCodeLines() const { - if (errPos.line <= 0) + if (line == 0) return std::nullopt; - if (errPos.origin == foFile) { - LinesOfCode loc; - try { - // FIXME: when running as the daemon, make sure we don't - // open a file to which the client doesn't have access. - AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) return {}; - - // count the newlines. - int count = 0; - std::string line; - int pl = errPos.line - 1; - do - { - line = readLine(fd.get()); - ++count; - if (count < pl) - ; - else if (count == pl) - loc.prevLineOfCode = line; - else if (count == pl + 1) - loc.errLineOfCode = line; - else if (count == pl + 2) { - loc.nextLineOfCode = line; - break; - } - } while (true); - return loc; - } - catch (EndOfFile & eof) { - if (loc.errLineOfCode.has_value()) - return loc; - else - return std::nullopt; - } - catch (std::exception & e) { - return std::nullopt; - } - } else { - std::istringstream iss(errPos.file); + if (auto source = getSource()) { + + std::istringstream iss(*source); // count the newlines. int count = 0; - std::string line; - int pl = errPos.line - 1; + std::string curLine; + int pl = line - 1; LinesOfCode loc; - do - { - std::getline(iss, line); + do { + std::getline(iss, curLine); ++count; if (count < pl) - { ; - } else if (count == pl) { - loc.prevLineOfCode = line; + loc.prevLineOfCode = curLine; } else if (count == pl + 1) { - loc.errLineOfCode = line; + loc.errLineOfCode = curLine; } else if (count == pl + 2) { - loc.nextLineOfCode = line; + loc.nextLineOfCode = curLine; break; } @@ -124,12 +79,14 @@ std::optional getCodeLines(const ErrPos & errPos) return loc; } + + return std::nullopt; } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, const std::string & prefix, - const ErrPos & errPos, + const AbstractPos & errPos, const LinesOfCode & loc) { // previous line of code. @@ -176,28 +133,6 @@ void printCodeLines(std::ostream & out, } } -void printAtPos(const ErrPos & pos, std::ostream & out) -{ - if (pos) { - switch (pos.origin) { - case foFile: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos)); - break; - } - case foString: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos)); - break; - } - case foStdin: { - out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos)); - break; - } - default: - throw Error("invalid FileOrigin in errPos"); - } - } -} - static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s) { std::string res; @@ -263,22 +198,22 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s std::ostringstream oss; + auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; + // traces if (showTrace && !einfo.traces.empty()) { for (const auto & trace : einfo.traces) { oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos.has_value() && (*trace.pos)) { - auto pos = trace.pos.value(); - oss << "\n"; - printAtPos(pos, oss); + if (trace.pos) { + oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; - auto loc = getCodeLines(pos); - if (loc.has_value()) { + if (auto loc = trace.pos->getCodeLines()) { oss << "\n"; - printCodeLines(oss, "", pos, *loc); + printCodeLines(oss, "", *trace.pos, *loc); oss << "\n"; - } + } else + oss << noSource; } } oss << "\n" << prefix; @@ -286,22 +221,19 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << einfo.msg << "\n"; - if (einfo.errPos.has_value() && *einfo.errPos) { - oss << "\n"; - printAtPos(*einfo.errPos, oss); - - auto loc = getCodeLines(*einfo.errPos); + if (einfo.errPos) { + oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - // lines of code. - if (loc.has_value()) { + if (auto loc = einfo.errPos->getCodeLines()) { oss << "\n"; printCodeLines(oss, "", *einfo.errPos, *loc); oss << "\n"; - } + } else + oss << noSource; } auto suggestions = einfo.suggestions.trim(); - if (! suggestions.suggestions.empty()){ + if (!suggestions.suggestions.empty()) { oss << "Did you mean " << suggestions.trim() << "?" << std::endl; diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 3d1479c54..c3bb8c0df 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -54,13 +54,6 @@ typedef enum { lvlVomit } Verbosity; -/* adjust Pos::origin bit width when adding stuff here */ -typedef enum { - foFile, - foStdin, - foString -} FileOrigin; - // the lines of code surrounding an error. struct LinesOfCode { std::optional prevLineOfCode; @@ -68,54 +61,37 @@ struct LinesOfCode { std::optional nextLineOfCode; }; -// ErrPos indicates the location of an error in a nix file. -struct ErrPos { - int line = 0; - int column = 0; - std::string file; - FileOrigin origin; +/* An abstract type that represents a location in a source file. */ +struct AbstractPos +{ + uint32_t line = 0; + uint32_t column = 0; - operator bool() const - { - return line != 0; - } + /* Return the contents of the source file. */ + virtual std::optional getSource() const + { return std::nullopt; }; - // convert from the Pos struct, found in libexpr. - template - ErrPos & operator=(const P & pos) - { - origin = pos.origin; - line = pos.line; - column = pos.column; - file = pos.file; - return *this; - } + virtual void print(std::ostream & out) const = 0; - template - ErrPos(const P & p) - { - *this = p; - } + std::optional getCodeLines() const; }; -std::optional getCodeLines(const ErrPos & errPos); +std::ostream & operator << (std::ostream & str, const AbstractPos & pos); void printCodeLines(std::ostream & out, const std::string & prefix, - const ErrPos & errPos, + const AbstractPos & errPos, const LinesOfCode & loc); -void printAtPos(const ErrPos & pos, std::ostream & out); - struct Trace { - std::optional pos; + std::shared_ptr pos; hintformat hint; }; struct ErrorInfo { Verbosity level; hintformat msg; - std::optional errPos; + std::shared_ptr errPos; std::list traces; Suggestions suggestions; @@ -177,12 +153,12 @@ public: const ErrorInfo & info() const { calcWhat(); return err; } template - void addTrace(std::optional e, const std::string & fs, const Args & ... args) + void addTrace(std::shared_ptr && e, const std::string & fs, const Args & ... args) { - addTrace(e, hintfmt(fs, args...)); + addTrace(std::move(e), hintfmt(fs, args...)); } - void addTrace(std::optional e, hintformat hint); + void addTrace(std::shared_ptr && e, hintformat hint); bool hasTrace() const { return !err.traces.empty(); } }; diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index ac86d8ac2..904ba6ebe 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -131,6 +131,21 @@ Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, logger.startActivity(id, lvl, type, s, fields, parent); } +void to_json(nlohmann::json & json, std::shared_ptr pos) +{ + if (pos) { + json["line"] = pos->line; + json["column"] = pos->column; + std::ostringstream str; + pos->print(str); + json["file"] = str.str(); + } else { + json["line"] = nullptr; + json["column"] = nullptr; + json["file"] = nullptr; + } +} + struct JSONLogger : Logger { Logger & prevLogger; @@ -177,27 +192,14 @@ struct JSONLogger : Logger { json["level"] = ei.level; json["msg"] = oss.str(); json["raw_msg"] = ei.msg.str(); - - if (ei.errPos.has_value() && (*ei.errPos)) { - json["line"] = ei.errPos->line; - json["column"] = ei.errPos->column; - json["file"] = ei.errPos->file; - } else { - json["line"] = nullptr; - json["column"] = nullptr; - json["file"] = nullptr; - } + to_json(json, ei.errPos); if (loggerSettings.showTrace.get() && !ei.traces.empty()) { nlohmann::json traces = nlohmann::json::array(); for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) { nlohmann::json stackFrame; stackFrame["raw_msg"] = iter->hint.str(); - if (iter->pos.has_value() && (*iter->pos)) { - stackFrame["line"] = iter->pos->line; - stackFrame["column"] = iter->pos->column; - stackFrame["file"] = iter->pos->file; - } + to_json(stackFrame, iter->pos); traces.push_back(stackFrame); } -- cgit v1.2.3 From 64c60f7241217cf4ba652f6502935ac7d523fb55 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 Dec 2022 15:32:54 +0100 Subject: Fix CanonPath::dirOf() returning a string_view of a temporary https://hydra.nixos.org/build/202837872 --- src/libutil/canon-path.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index c5e7f0596..9d5984584 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -110,7 +110,7 @@ public: std::optional dirOf() const { if (isRoot()) return std::nullopt; - return path.substr(0, path.rfind('/')); + return ((std::string_view) path).substr(0, path.rfind('/')); } std::optional baseName() const -- cgit v1.2.3 From d33d15a48b55eb81270a39b001c04bc61c42105f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Jan 2023 20:40:39 +0100 Subject: Put the --show-trace hint in the logical place --- src/libutil/error.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libutil') diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 7f7c27267..e4f0d4677 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -303,7 +303,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s size_t count = 0; for (const auto & trace : einfo.traces) { if (!showTrace && count > 3) { - oss << "\n" << ANSI_ITALIC "(stack trace truncated)" ANSI_NORMAL << "\n"; + oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; break; } -- cgit v1.2.3