diff options
Diffstat (limited to 'src/libutil')
42 files changed, 1267 insertions, 678 deletions
diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index b3fbc84f7..2d82b54e7 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -10,6 +10,7 @@ std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject() auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); obj.emplace("defaultValue", defaultValue); + obj.emplace("documentDefault", documentDefault); return obj; } } diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc deleted file mode 100644 index ac2295e4a..000000000 --- a/src/libutil/affinity.cc +++ /dev/null @@ -1,70 +0,0 @@ -#include "types.hh" -#include "util.hh" -#include "affinity.hh" - -#if __linux__ -#include <sched.h> -#endif - -namespace nix { - - -#if __linux__ -static bool didSaveAffinity = false; -static cpu_set_t savedAffinity; - -std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset) -{ - auto count = CPU_COUNT(&cset); - for (int i=0; i < count; ++i) - { - os << (CPU_ISSET(i,&cset) ? "1" : "0"); - } - - return os; -} -#endif - - -void setAffinityTo(int cpu) -{ -#if __linux__ - if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; - didSaveAffinity = true; - debug(format("locking this thread to CPU %1%") % cpu); - cpu_set_t newAffinity; - CPU_ZERO(&newAffinity); - CPU_SET(cpu, &newAffinity); - if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printError("failed to lock thread to CPU %1%", cpu); -#endif -} - - -int lockToCurrentCPU() -{ -#if __linux__ - int cpu = sched_getcpu(); - if (cpu != -1) setAffinityTo(cpu); - return cpu; -#else - return -1; -#endif -} - - -void restoreAffinity() -{ -#if __linux__ - if (!didSaveAffinity) return; - if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - { - std::ostringstream oss; - oss << savedAffinity; - printError("failed to restore CPU affinity %1%", oss.str()); - } -#endif -} - - -} diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh deleted file mode 100644 index c1bd28e13..000000000 --- a/src/libutil/affinity.hh +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -namespace nix { - -void setAffinityTo(int cpu); -int lockToCurrentCPU(); -void restoreAffinity(); - -} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index ed0eb2fb5..eda004756 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -37,12 +37,12 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); const std::string narVersionMagic1 = "nix-archive-1"; -static string caseHackSuffix = "~nix~case~hack~"; +static std::string caseHackSuffix = "~nix~case~hack~"; PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, size_t size, +static void dumpContents(const Path & path, off_t size, Sink & sink) { sink << "contents" << size; @@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) sink << "type" << "regular"; if (st.st_mode & S_IXUSR) sink << "executable" << ""; - dumpContents(path, (size_t) st.st_size, sink); + dumpContents(path, st.st_size, sink); } else if (S_ISDIR(st.st_mode)) { @@ -84,22 +84,21 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) /* If we're on a case-insensitive system like macOS, undo the case hack applied by restorePath(). */ - std::map<string, string> unhacked; + std::map<std::string, std::string> unhacked; for (auto & i : readDirectory(path)) if (archiveSettings.useCaseHack) { - string name(i.name); + std::string name(i.name); size_t pos = i.name.find(caseHackSuffix); - if (pos != string::npos) { + if (pos != std::string::npos) { debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); name.erase(pos); } - if (unhacked.find(name) != unhacked.end()) + if (!unhacked.emplace(name, i.name).second) throw Error("file name collision in between '%1%' and '%2%'", (path + "/" + unhacked[name]), (path + "/" + i.name)); - unhacked[name] = i.name; } else - unhacked[i.name] = i.name; + unhacked.emplace(i.name, i.name); for (auto & i : unhacked) if (filter(path + "/" + i.first)) { @@ -125,13 +124,13 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter) } -void dumpString(const std::string & s, Sink & sink) +void dumpString(std::string_view s, Sink & sink) { sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; } -static SerialisationError badArchive(string s) +static SerialisationError badArchive(const std::string & s) { return SerialisationError("bad archive: " + s); } @@ -172,7 +171,7 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) struct CaseInsensitiveCompare { - bool operator() (const string & a, const string & b) const + bool operator() (const std::string & a, const std::string & b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; } @@ -181,7 +180,7 @@ struct CaseInsensitiveCompare static void parse(ParseSink & sink, Source & source, const Path & path) { - string s; + std::string s; s = readString(source); if (s != "(") throw badArchive("expected open tag"); @@ -202,7 +201,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) else if (s == "type") { if (type != tpUnknown) throw badArchive("multiple type fields"); - string t = readString(source); + std::string t = readString(source); if (t == "regular") { type = tpRegular; @@ -233,7 +232,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "entry" && type == tpDirectory) { - string name, prevName; + std::string name, prevName; s = readString(source); if (s != "(") throw badArchive("expected open tag"); @@ -247,7 +246,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) break; } else if (s == "name") { name = readString(source); - if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) + if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) throw Error("NAR contains invalid file name '%1%'", name); if (name <= prevName) throw Error("NAR directory is not sorted"); @@ -270,7 +269,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "target" && type == tpSymlink) { - string target = readString(source); + std::string target = readString(source); sink.createSymlink(path, target); } @@ -282,7 +281,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) void parseDump(ParseSink & sink, Source & source) { - string version; + std::string version; try { version = readString(source, narVersionMagic1.size()); } catch (SerialisationError & e) { @@ -346,7 +345,7 @@ struct RestoreSink : ParseSink writeFull(fd.get(), data); } - void createSymlink(const Path & path, const string & target) override + void createSymlink(const Path & path, const std::string & target) override { Path p = dstPath + path; nix::createSymlink(target, p); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 9e9e11b1a..fca351605 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -48,7 +48,7 @@ namespace nix { void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); -void dumpString(const std::string & s, Sink & sink); +void dumpString(std::string_view s, Sink & sink); /* FIXME: fix this API, it sucks. */ struct ParseSink @@ -60,7 +60,7 @@ struct ParseSink virtual void preallocateContents(uint64_t size) { }; virtual void receiveContents(std::string_view data) { }; - virtual void createSymlink(const Path & path, const string & target) { }; + virtual void createSymlink(const Path & path, const std::string & target) { }; }; /* If the NAR archive contains a single file at top-level, then save @@ -82,7 +82,7 @@ struct RetrieveRegularNARSink : ParseSink sink(data); } - void createSymlink(const Path & path, const string & target) override + void createSymlink(const Path & path, const std::string & target) override { regular = false; } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 9df279faf..69aa0d094 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -39,7 +39,7 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -bool pathCompletions = false; +CompletionType completionType = ctNormal; std::shared_ptr<Completions> completions; std::string completionMarker = "___COMPLETE___"; @@ -76,13 +76,13 @@ void Args::parseCmdline(const Strings & _cmdline) /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', `-j3` -> `-j 3`). */ if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { - *pos = (string) "-" + arg[1]; + *pos = (std::string) "-" + arg[1]; auto next = pos; ++next; for (unsigned int j = 2; j < arg.length(); j++) if (isalpha(arg[j])) - cmdline.insert(next, (string) "-" + arg[j]); + cmdline.insert(next, (std::string) "-" + arg[j]); else { - cmdline.insert(next, string(arg, j)); + cmdline.insert(next, std::string(arg, j)); break; } arg = *pos; @@ -139,7 +139,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return true; }; - if (string(*pos, 0, 2) == "--") { + if (std::string(*pos, 0, 2) == "--") { if (auto prefix = needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) @@ -147,12 +147,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) completions->add("--" + name, flag->description); } } - auto i = longFlags.find(string(*pos, 2)); + auto i = longFlags.find(std::string(*pos, 2)); if (i == longFlags.end()) return false; return process("--" + i->first, *i->second); } - if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + if (std::string(*pos, 0, 1) == "-" && pos->size() == 2) { auto c = (*pos)[1]; auto i = shortFlags.find(c); if (i == shortFlags.end()) return false; @@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< static void _completePath(std::string_view prefix, bool onlyDirs) { - pathCompletions = true; + completionType = ctFilenames; glob_t globbuf; int flags = GLOB_NOESCAPE | GLOB_TILDE; #ifdef GLOB_ONLYDIR @@ -328,8 +328,13 @@ MultiCommand::MultiCommand(const Commands & commands_) completions->add(name); } auto i = commands.find(s); - if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", s); + if (i == commands.end()) { + std::set<std::string> commandNames; + for (auto & [name, _] : commands) + commandNames.insert(name); + auto suggestions = Suggestions::bestMatches(commandNames, s); + throw UsageError(suggestions, "'%s' is not a recognised command", s); + } command = {s, i->second()}; command->second->parent = this; }} diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7521b3065..fdd036f9a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -158,7 +158,7 @@ public: } /* Expect a string argument. */ - void expectArg(const std::string & label, string * dest, bool optional = false) + void expectArg(const std::string & label, std::string * dest, bool optional = false) { expectArgs({ .label = label, @@ -237,7 +237,13 @@ public: void add(std::string completion, std::string description = ""); }; extern std::shared_ptr<Completions> completions; -extern bool pathCompletions; + +enum CompletionType { + ctNormal, + ctFilenames, + ctAttrs +}; +extern CompletionType completionType; std::optional<std::string> needsCompletion(std::string_view s); diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index d26f68fde..89180e7a7 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -16,6 +16,8 @@ namespace nix { +static const int COMPRESSION_LEVEL_DEFAULT = -1; + // Don't feed brotli too much at once. struct ChunkedCompressionSink : CompressionSink { @@ -65,14 +67,16 @@ struct ArchiveCompressionSink : CompressionSink Sink & nextSink; struct archive * archive; - ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) { + ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + { archive = archive_write_new(); if (!archive) throw Error("failed to initialize libarchive"); check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); check(archive_write_set_format_raw(archive)); - if (format == "xz" && parallel) { + if (parallel) check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); - } + if (level != COMPRESSION_LEVEL_DEFAULT) + check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str())); // disable internal buffering check(archive_write_set_bytes_per_block(archive, 0)); // disable output padding @@ -126,7 +130,11 @@ private: struct NoneSink : CompressionSink { Sink & nextSink; - NoneSink(Sink & nextSink) : nextSink(nextSink) { } + NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + { + if (level != COMPRESSION_LEVEL_DEFAULT) + warn("requested compression level '%d' not supported by compression method 'none'", level); + } void finish() override { flush(); } void write(std::string_view data) override { nextSink(data); } }; @@ -182,13 +190,13 @@ struct BrotliDecompressionSink : ChunkedCompressionSink } }; -ref<std::string> decompress(const std::string & method, const std::string & in) +std::string decompress(const std::string & method, std::string_view in) { StringSink ssink; auto sink = makeDecompressionSink(method, ssink); (*sink)(in); sink->finish(); - return ssink.s; + return std::move(ssink.s); } std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink) @@ -257,13 +265,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink } }; -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { std::vector<std::string> la_supports = { "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" }; if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { - return make_ref<ArchiveCompressionSink>(nextSink, method, parallel); + return make_ref<ArchiveCompressionSink>(nextSink, method, parallel, level); } if (method == "none") return make_ref<NoneSink>(nextSink); @@ -273,13 +281,13 @@ ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & next throw UnknownCompressionMethod("unknown compression method '%s'", method); } -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel) +std::string compress(const std::string & method, std::string_view in, const bool parallel, int level) { StringSink ssink; - auto sink = makeCompressionSink(method, ssink, parallel); + auto sink = makeCompressionSink(method, ssink, parallel, level); (*sink)(in); sink->finish(); - return ssink.s; + return std::move(ssink.s); } } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 338a0d9f2..c470b82a5 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -15,13 +15,13 @@ struct CompressionSink : BufferedSink, FinishSink using FinishSink::finish; }; -ref<std::string> decompress(const std::string & method, const std::string & in); +std::string decompress(const std::string & method, std::string_view in); std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink); -ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false); +std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1); -ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); +ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index c247c7dae..9bb412b4f 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "experimental-features.hh" #include <nlohmann/json.hpp> @@ -80,16 +81,16 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string unsigned int pos = 0; while (pos < contents.size()) { - string line; + std::string line; while (pos < contents.size() && contents[pos] != '\n') line += contents[pos++]; pos++; - string::size_type hash = line.find('#'); - if (hash != string::npos) - line = string(line, 0, hash); + auto hash = line.find('#'); + if (hash != std::string::npos) + line = std::string(line, 0, hash); - vector<string> tokens = tokenizeString<vector<string> >(line); + auto tokens = tokenizeString<std::vector<std::string>>(line); if (tokens.empty()) continue; if (tokens.size() < 2) @@ -119,9 +120,9 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - string name = tokens[0]; + std::string name = tokens[0]; - vector<string>::iterator i = tokens.begin(); + auto i = tokens.begin(); advance(i, 2); set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow @@ -131,7 +132,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string void AbstractConfig::applyConfigFile(const Path & path) { try { - string contents = readFile(path); + std::string contents = readFile(path); applyConfig(contents, path); } catch (SysError &) { } } @@ -313,6 +314,31 @@ template<> std::string BaseSetting<StringSet>::to_string() const return concatStringsSep(" ", value); } +template<> void BaseSetting<std::set<ExperimentalFeature>>::set(const std::string & str, bool append) +{ + if (!append) value.clear(); + for (auto & s : tokenizeString<StringSet>(str)) { + auto thisXpFeature = parseExperimentalFeature(s); + if (thisXpFeature) + value.insert(thisXpFeature.value()); + else + warn("unknown experimental feature '%s'", s); + } +} + +template<> bool BaseSetting<std::set<ExperimentalFeature>>::isAppendable() +{ + return true; +} + +template<> std::string BaseSetting<std::set<ExperimentalFeature>>::to_string() const +{ + StringSet stringifiedXpFeatures; + for (auto & feature : value) + stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); + return concatStringsSep(" ", stringifiedXpFeatures); +} + template<> void BaseSetting<StringMap>::set(const std::string & str, bool append) { if (!append) value.clear(); @@ -348,6 +374,7 @@ template class BaseSetting<std::string>; template class BaseSetting<Strings>; template class BaseSetting<StringSet>; template class BaseSetting<StringMap>; +template class BaseSetting<std::set<ExperimentalFeature>>; void PathSetting::set(const std::string & str, bool append) { diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 736810bf3..79ec0f9cf 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -232,16 +232,19 @@ protected: T value; const T defaultValue; + const bool documentDefault; public: BaseSetting(const T & def, + const bool documentDefault, const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}) : AbstractSetting(name, description, aliases) , value(def) , defaultValue(def) + , documentDefault(documentDefault) { } operator const T &() const { return value; } @@ -288,8 +291,9 @@ public: const T & def, const std::string & name, const std::string & description, - const std::set<std::string> & aliases = {}) - : BaseSetting<T>(def, name, description, aliases) + const std::set<std::string> & aliases = {}, + const bool documentDefault = true) + : BaseSetting<T>(def, documentDefault, name, description, aliases) { options->addSetting(this); } @@ -311,7 +315,7 @@ public: const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}) - : BaseSetting<Path>(def, name, description, aliases) + : BaseSetting<Path>(def, true, name, description, aliases) , allowEmpty(allowEmpty) { options->addSetting(this); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 203d79087..b2dfb35b2 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -17,7 +17,7 @@ BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). -const string & BaseError::calcWhat() const +const std::string & BaseError::calcWhat() const { if (what_.has_value()) return *what_; @@ -25,21 +25,21 @@ const string & BaseError::calcWhat() const err.name = sname(); std::ostringstream oss; - showErrorInfo(oss, err, false); + showErrorInfo(oss, err, loggerSettings.showTrace); what_ = oss.str(); return *what_; } } -std::optional<string> ErrorInfo::programName = std::nullopt; +std::optional<std::string> ErrorInfo::programName = std::nullopt; std::ostream & operator<<(std::ostream & os, const hintformat & hf) { return os << hf.str(); } -string showErrPos(const ErrPos & errPos) +std::string showErrPos(const ErrPos & errPos) { if (errPos.line > 0) { if (errPos.column > 0) { @@ -68,7 +68,7 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) // count the newlines. int count = 0; - string line; + std::string line; int pl = errPos.line - 1; do { @@ -100,7 +100,7 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) std::istringstream iss(errPos.file); // count the newlines. int count = 0; - string line; + std::string line; int pl = errPos.line - 1; LinesOfCode loc; @@ -132,7 +132,7 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, - const string & prefix, + const std::string & prefix, const ErrPos & errPos, const LinesOfCode & loc) { @@ -282,6 +282,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s } } + auto suggestions = einfo.suggestions.trim(); + if (! suggestions.suggestions.empty()){ + oss << "Did you mean " << + suggestions.trim() << + "?" << std::endl; + } + // traces if (showTrace && !einfo.traces.empty()) { for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index e55986d6f..cb6532587 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,5 +1,6 @@ #pragma once +#include "suggestions.hh" #include "ref.hh" #include "types.hh" #include "fmt.hh" @@ -61,16 +62,16 @@ typedef enum { // the lines of code surrounding an error. struct LinesOfCode { - std::optional<string> prevLineOfCode; - std::optional<string> errLineOfCode; - std::optional<string> nextLineOfCode; + std::optional<std::string> prevLineOfCode; + std::optional<std::string> errLineOfCode; + std::optional<std::string> nextLineOfCode; }; // ErrPos indicates the location of an error in a nix file. struct ErrPos { int line = 0; int column = 0; - string file; + std::string file; FileOrigin origin; operator bool() const @@ -80,7 +81,7 @@ struct ErrPos { // convert from the Pos struct, found in libexpr. template <class P> - ErrPos& operator=(const P &pos) + ErrPos & operator=(const P & pos) { origin = pos.origin; line = pos.line; @@ -94,7 +95,7 @@ struct ErrPos { } template <class P> - ErrPos(const P &p) + ErrPos(const P & p) { *this = p; } @@ -107,15 +108,17 @@ struct Trace { struct ErrorInfo { Verbosity level; - string name; // FIXME: rename + std::string name; // FIXME: rename hintformat msg; std::optional<ErrPos> errPos; std::list<Trace> traces; - static std::optional<string> programName; + Suggestions suggestions; + + static std::optional<std::string> programName; }; -std::ostream& showErrorInfo(std::ostream &out, const ErrorInfo &einfo, bool showTrace); +std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace); /* BaseError should generally not be caught, as it has Interrupted as a subclass. Catch Error instead. */ @@ -124,8 +127,8 @@ class BaseError : public std::exception protected: mutable ErrorInfo err; - mutable std::optional<string> what_; - const string& calcWhat() const; + mutable std::optional<std::string> what_; + const std::string & calcWhat() const; public: unsigned int status = 1; // exit status @@ -137,10 +140,15 @@ public: { } template<typename... Args> - BaseError(const std::string & fs, const Args & ... args) + explicit BaseError(const std::string & fs, const Args & ... args) : err { .level = lvlError, .msg = hintfmt(fs, args...) } { } + template<typename... Args> + BaseError(const Suggestions & sug, const Args & ... args) + : err { .level = lvlError, .msg = hintfmt(args...), .suggestions = sug } + { } + BaseError(hintformat hint) : err { .level = lvlError, .msg = hint } { } @@ -162,11 +170,11 @@ public: const char * what() const noexcept override { return calcWhat().c_str(); } #endif - const string & msg() const { return calcWhat(); } + const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } template<typename... Args> - BaseError & addTrace(std::optional<ErrPos> e, const string &fs, const Args & ... args) + BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) { return addTrace(e, hintfmt(fs, args...)); } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc new file mode 100644 index 000000000..b49f47e1d --- /dev/null +++ b/src/libutil/experimental-features.cc @@ -0,0 +1,59 @@ +#include "experimental-features.hh" +#include "util.hh" + +#include "nlohmann/json.hpp" + +namespace nix { + +std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { + { Xp::CaDerivations, "ca-derivations" }, + { Xp::Flakes, "flakes" }, + { Xp::NixCommand, "nix-command" }, + { Xp::RecursiveNix, "recursive-nix" }, + { Xp::NoUrlLiterals, "no-url-literals" }, +}; + +const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name) +{ + using ReverseXpMap = std::map<std::string_view, ExperimentalFeature>; + + static auto reverseXpMap = []() + { + auto reverseXpMap = std::make_unique<ReverseXpMap>(); + for (auto & [feature, name] : stringifiedXpFeatures) + (*reverseXpMap)[name] = feature; + return reverseXpMap; + }(); + + if (auto feature = get(*reverseXpMap, name)) + return *feature; + else + return std::nullopt; +} + +std::string_view showExperimentalFeature(const ExperimentalFeature feature) +{ + return stringifiedXpFeatures.at(feature); +} + +std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures) +{ + std::set<ExperimentalFeature> res; + for (auto & rawFeature : rawFeatures) { + if (auto feature = parseExperimentalFeature(rawFeature)) + res.insert(*feature); + } + return res; +} + +MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) + : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + , missingFeature(feature) +{} + +std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & feature) +{ + return str << showExperimentalFeature(feature); +} + +} diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh new file mode 100644 index 000000000..291a58e32 --- /dev/null +++ b/src/libutil/experimental-features.hh @@ -0,0 +1,56 @@ +#pragma once + +#include "comparator.hh" +#include "error.hh" +#include "nlohmann/json_fwd.hpp" +#include "types.hh" + +namespace nix { + +/** + * The list of available experimental features. + * + * If you update this, don’t forget to also change the map defining their + * string representation in the corresponding `.cc` file. + **/ +enum struct ExperimentalFeature +{ + CaDerivations, + Flakes, + NixCommand, + RecursiveNix, + NoUrlLiterals +}; + +/** + * Just because writing `ExperimentalFeature::CaDerivations` is way too long + */ +using Xp = ExperimentalFeature; + +const std::optional<ExperimentalFeature> parseExperimentalFeature( + const std::string_view & name); +std::string_view showExperimentalFeature(const ExperimentalFeature); + +std::ostream & operator<<( + std::ostream & str, + const ExperimentalFeature & feature); + +/** + * Parse a set of strings to the corresponding set of experimental features, + * ignoring (but warning for) any unkwown feature. + */ +std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> &); + +class MissingExperimentalFeature : public Error +{ +public: + ExperimentalFeature missingFeature; + + MissingExperimentalFeature(ExperimentalFeature); + virtual const char * sname() const override + { + return "MissingExperimentalFeature"; + } +}; + +} diff --git a/src/libutil/fmt.cc b/src/libutil/fmt.cc new file mode 100644 index 000000000..3dd93d73e --- /dev/null +++ b/src/libutil/fmt.cc @@ -0,0 +1,46 @@ +#include "fmt.hh" + +#include <regex> + +namespace nix { + +std::string hiliteMatches( + std::string_view s, + std::vector<std::smatch> matches, + std::string_view prefix, + std::string_view postfix) +{ + // Avoid copy on zero matches + if (matches.size() == 0) + return (std::string) s; + + std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) { + return a.position() < b.position(); + }); + + std::string out; + ssize_t last_end = 0; + + for (auto it = matches.begin(); it != matches.end();) { + auto m = *it; + size_t start = m.position(); + out.append(s.substr(last_end, m.position() - last_end)); + // Merge continous matches + ssize_t end = start + m.length(); + while (++it != matches.end() && (*it).position() <= end) { + auto n = *it; + ssize_t nend = start + (n.position() - start + n.length()); + if (nend > end) + end = nend; + } + out.append(prefix); + out.append(s.substr(start, end - start)); + out.append(postfix); + last_end = end; + } + + out.append(s.substr(last_end)); + return out; +} + +} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index fd335b811..0821b3b74 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -2,6 +2,7 @@ #include <boost/format.hpp> #include <string> +#include <regex> #include "ansicolor.hh" @@ -9,7 +10,6 @@ namespace nix { /* Inherit some names from other namespaces for convenience. */ -using std::string; using boost::format; @@ -20,8 +20,8 @@ struct nop { template<typename... T> nop(T...) {} }; struct FormatOrString { - string s; - FormatOrString(const string & s) : s(s) { }; + std::string s; + FormatOrString(std::string s) : s(std::move(s)) { }; template<class F> FormatOrString(const F & f) : s(f.str()) { }; FormatOrString(const char * s) : s(s) { }; @@ -101,7 +101,7 @@ std::ostream & operator<<(std::ostream & out, const normaltxt<T> & y) class hintformat { public: - hintformat(const string & format) : fmt(format) + hintformat(const std::string & format) : fmt(format) { fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ @@ -154,4 +154,16 @@ inline hintformat hintfmt(std::string plain_string) // we won't be receiving any args in this case, so just print the original string return hintfmt("%s", normaltxt(plain_string)); } + +/* Highlight all the given matches in the given string `s` by wrapping + them between `prefix` and `postfix`. + + If some matches overlap, then their union will be wrapped rather + than the individual matches. */ +std::string hiliteMatches( + std::string_view s, + std::vector<std::smatch> matches, + std::string_view prefix, + std::string_view postfix); + } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 4df8b4ecb..a4d632161 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -66,31 +66,31 @@ bool Hash::operator < (const Hash & h) const } -const string base16Chars = "0123456789abcdef"; +const std::string base16Chars = "0123456789abcdef"; -static string printHash16(const Hash & hash) +static std::string printHash16(const Hash & hash) { char buf[hash.hashSize * 2]; for (unsigned int i = 0; i < hash.hashSize; i++) { buf[i * 2] = base16Chars[hash.hash[i] >> 4]; buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; } - return string(buf, hash.hashSize * 2); + return std::string(buf, hash.hashSize * 2); } // omitted: E O U T -const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; +const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; -static string printHash32(const Hash & hash) +static std::string printHash32(const Hash & hash) { assert(hash.hashSize); size_t len = hash.base32Len(); assert(len); - string s; + std::string s; s.reserve(len); for (int n = (int) len - 1; n >= 0; n--) { @@ -107,7 +107,7 @@ static string printHash32(const Hash & hash) } -string printHash16or32(const Hash & hash) +std::string printHash16or32(const Hash & hash) { assert(hash.type); return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); @@ -151,7 +151,8 @@ Hash Hash::parseSRI(std::string_view original) { } // Mutates the string to eliminate the prefixes when found -static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest) { +static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_view & rest) +{ bool isSRI = false; // Parse the has type before the separater, if there was one. @@ -259,7 +260,7 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); } -Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht) +Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht) { if (hashStr.empty()) { if (!ht) @@ -402,7 +403,7 @@ HashType parseHashType(std::string_view s) throw UsageError("unknown hash algorithm '%1%'", s); } -string printHashType(HashType ht) +std::string printHashType(HashType ht) { switch (ht) { case htMD5: return "md5"; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 1b626dd85..56b5938b3 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,7 +20,7 @@ const int sha512HashSize = 64; extern std::set<std::string> hashTypes; -extern const string base32Chars; +extern const std::string base32Chars; enum Base : int { Base64, Base32, Base16, SRI }; @@ -107,10 +107,10 @@ public: }; /* Helper that defaults empty hashes to the 0 hash. */ -Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht); +Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht); /* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ -string printHash16or32(const Hash & hash); +std::string printHash16or32(const Hash & hash); /* Compute the hash of the given string. */ Hash hashString(HashType ht, std::string_view s); @@ -135,7 +135,7 @@ HashType parseHashType(std::string_view s); std::optional<HashType> parseHashTypeOpt(std::string_view s); /* And the reverse. */ -string printHashType(HashType ht); +std::string printHashType(HashType ht); union Ctx; diff --git a/src/libutil/json.cc b/src/libutil/json.cc index 01331947e..3a981376f 100644 --- a/src/libutil/json.cc +++ b/src/libutil/json.cc @@ -7,16 +7,38 @@ namespace nix { void toJSON(std::ostream & str, const char * start, const char * end) { - str << '"'; - for (auto i = start; i != end; i++) - if (*i == '\"' || *i == '\\') str << '\\' << *i; - else if (*i == '\n') str << "\\n"; - else if (*i == '\r') str << "\\r"; - else if (*i == '\t') str << "\\t"; - else if (*i >= 0 && *i < 32) - str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec; - else str << *i; - str << '"'; + constexpr size_t BUF_SIZE = 4096; + char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts + size_t bufPos = 0; + + const auto flush = [&] { + str.write(buf, bufPos); + bufPos = 0; + }; + const auto put = [&] (char c) { + buf[bufPos++] = c; + }; + + put('"'); + for (auto i = start; i != end; i++) { + if (bufPos >= BUF_SIZE) flush(); + if (*i == '\"' || *i == '\\') { put('\\'); put(*i); } + else if (*i == '\n') { put('\\'); put('n'); } + else if (*i == '\r') { put('\\'); put('r'); } + else if (*i == '\t') { put('\\'); put('t'); } + else if (*i >= 0 && *i < 32) { + const char hex[17] = "0123456789abcdef"; + put('\\'); + put('u'); + put(hex[(uint16_t(*i) >> 12) & 0xf]); + put(hex[(uint16_t(*i) >> 8) & 0xf]); + put(hex[(uint16_t(*i) >> 4) & 0xf]); + put(hex[(uint16_t(*i) >> 0) & 0xf]); + } + else put(*i); + } + put('"'); + flush(); } void toJSON(std::ostream & str, const char * s) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index f8a121ed1..cb2b15b41 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -113,7 +113,7 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs) } } -void writeToStderr(const string & s) +void writeToStderr(std::string_view s) { try { writeFull(STDERR_FILENO, s, false); @@ -266,51 +266,63 @@ static Logger::Fields getFields(nlohmann::json & json) return fields; } -bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) +std::optional<nlohmann::json> parseJSONMessage(const std::string & msg) { - if (!hasPrefix(msg, "@nix ")) return false; - + if (!hasPrefix(msg, "@nix ")) return std::nullopt; try { - auto json = nlohmann::json::parse(std::string(msg, 5)); - - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actFileTransfer) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); - } + return nlohmann::json::parse(std::string(msg, 5)); + } catch (std::exception & e) { + printError("bad JSON log message from builder: %s", e.what()); + } + return std::nullopt; +} - else if (action == "stop") - activities.erase((ActivityId) json["id"]); +bool handleJSONLogMessage(nlohmann::json & json, + const Activity & act, std::map<ActivityId, Activity> & activities, + bool trusted) +{ + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } + else if (action == "stop") + activities.erase((ActivityId) json["id"]); - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } - } catch (std::exception & e) { - printError("bad JSON log message from builder: %s", e.what()); + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); } return true; } +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) +{ + auto json = parseJSONMessage(msg); + if (!json) return false; + + return handleJSONLogMessage(*json, act, activities, trusted); +} + Activity::~Activity() { try { diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 96ad69790..6f81b92de 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -4,6 +4,8 @@ #include "error.hh" #include "config.hh" +#include <nlohmann/json_fwd.hpp> + namespace nix { typedef enum { @@ -40,7 +42,7 @@ struct LoggerSettings : Config Setting<bool> showTrace{ this, false, "show-trace", R"( - Where Nix should print out a stack trace in case of Nix + Whether Nix should print out a stack trace in case of Nix expression evaluation errors. )"}; }; @@ -166,6 +168,12 @@ Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); +std::optional<nlohmann::json> parseJSONMessage(const std::string & msg); + +bool handleJSONLogMessage(nlohmann::json & json, + const Activity & act, std::map<ActivityId, Activity> & activities, + bool trusted); + bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted); @@ -189,13 +197,14 @@ extern Verbosity verbosity; /* suppress msgs > this */ /* Print a string message if the current log level is at least the specified level. Note that this has to be implemented as a macro to ensure that the arguments are evaluated lazily. */ -#define printMsg(level, args...) \ +#define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ if (__lvl <= nix::verbosity) { \ - logger->log(__lvl, fmt(args)); \ + loggerParam->log(__lvl, fmt(args)); \ } \ } while (0) +#define printMsg(level, args...) printMsgUsing(logger, level, args) #define printError(args...) printMsg(lvlError, args) #define notice(args...) printMsg(lvlNotice, args) @@ -215,6 +224,6 @@ inline void warn(const std::string & fs, const Args & ... args) void warnOnce(bool & haveWarned, const FormatOrString & fs); -void writeToStderr(const string & s); +void writeToStderr(std::string_view s); } diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index d6bf53bb8..347b81f73 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -17,7 +17,7 @@ private: public: - ref<T>(const ref<T> & r) + ref(const ref<T> & r) : p(r.p) { } diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc deleted file mode 100644 index 67924568f..000000000 --- a/src/libutil/rust-ffi.cc +++ /dev/null @@ -1,24 +0,0 @@ -#if 0 -#include "logging.hh" -#include "rust-ffi.hh" - -extern "C" std::exception_ptr * make_error(rust::StringSlice s) -{ - return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size)))); -} - -extern "C" void destroy_error(std::exception_ptr * ex) -{ - free(ex); -} - -namespace rust { - -std::ostream & operator << (std::ostream & str, const String & s) -{ - str << (std::string_view) s; - return str; -} - -} -#endif diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh deleted file mode 100644 index cfbaf9dec..000000000 --- a/src/libutil/rust-ffi.hh +++ /dev/null @@ -1,189 +0,0 @@ -#pragma once -#if 0 - -#include "serialise.hh" - -#include <string_view> -#include <cstring> -#include <array> - -namespace rust { - -typedef void (*DropFun)(void *); - -/* A Rust value of N bytes. It can be moved but not copied. When it - goes out of scope, the C++ destructor will run the drop - function. */ -template<std::size_t N, DropFun drop> -struct Value -{ -protected: - - std::array<char, N> raw; - - ~Value() - { - if (!isEvacuated()) { - drop(this); - evacuate(); - } - } - - // Must not be called directly. - Value() - { } - - Value(Value && other) - : raw(other.raw) - { - other.evacuate(); - } - - void operator =(Value && other) - { - if (!isEvacuated()) - drop(this); - raw = other.raw; - other.evacuate(); - } - -private: - - /* FIXME: optimize these (ideally in such a way that the compiler - can elide most calls to evacuate() / isEvacuated(). */ - inline void evacuate() - { - for (auto & i : raw) i = 0; - } - - inline bool isEvacuated() - { - for (auto & i : raw) - if (i != 0) return false; - return true; - } -}; - -/* A Rust vector. */ -template<typename T, DropFun drop> -struct Vec : Value<3 * sizeof(void *), drop> -{ - inline size_t size() const - { - return ((const size_t *) &this->raw)[2]; - } - - const T * data() const - { - return ((const T * *) &this->raw)[0]; - } -}; - -/* A Rust slice. */ -template<typename T> -struct Slice -{ - const T * ptr; - size_t size; - - Slice(const T * ptr, size_t size) : ptr(ptr), size(size) - { - assert(ptr); - } -}; - -struct StringSlice : Slice<char> -{ - StringSlice(const std::string & s): Slice(s.data(), s.size()) {} - explicit StringSlice(std::string_view s): Slice(s.data(), s.size()) {} - StringSlice(const char * s): Slice(s, strlen(s)) {} - - operator std::string_view() const - { - return std::string_view(ptr, size); - } -}; - -/* A Rust string. */ -struct String; - -extern "C" { - void ffi_String_new(StringSlice s, String * out); - void ffi_String_drop(void * s); -} - -struct String : Vec<char, ffi_String_drop> -{ - String() = delete; - - String(std::string_view s) - { - ffi_String_new(StringSlice(s), this); - } - - String(const char * s) - : String({s, std::strlen(s)}) - { - } - - operator std::string_view() const - { - return std::string_view(data(), size()); - } -}; - -std::ostream & operator << (std::ostream & str, const String & s); - -/* C++ representation of Rust's Result<T, CppException>. */ -template<typename T> -struct Result -{ - enum { Ok = 0, Err = 1, Uninit = 2 } tag; - - union { - T data; - std::exception_ptr * exc; - }; - - Result() : tag(Uninit) { }; // FIXME: remove - - Result(const Result &) = delete; - - Result(Result && other) - : tag(other.tag) - { - other.tag = Uninit; - if (tag == Ok) - data = std::move(other.data); - else if (tag == Err) - exc = other.exc; - } - - ~Result() - { - if (tag == Ok) - data.~T(); - else if (tag == Err) - free(exc); - else if (tag == Uninit) - ; - else - abort(); - } - - /* Rethrow the wrapped exception or return the wrapped value. */ - T unwrap() - { - if (tag == Ok) { - tag = Uninit; - return std::move(data); - } - else if (tag == Err) - std::rethrow_exception(*exc); - else - abort(); - } -}; - -} -#endif diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 16f3476c2..6445b3f1b 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -110,7 +110,7 @@ std::string Source::drain() { StringSink s; drainInto(s); - return *s.s; + return std::move(s.s); } @@ -325,7 +325,7 @@ void writeString(std::string_view data, Sink & sink) } -Sink & operator << (Sink & sink, const string & s) +Sink & operator << (Sink & sink, std::string_view s) { writeString(s, sink); return sink; @@ -391,7 +391,7 @@ size_t readString(char * buf, size_t max, Source & source) } -string readString(Source & source, size_t max) +std::string readString(Source & source, size_t max) { auto len = readNum<size_t>(source); if (len > max) throw SerialisationError("string is too long"); @@ -401,7 +401,7 @@ string readString(Source & source, size_t max) return res; } -Source & operator >> (Source & in, string & s) +Source & operator >> (Source & in, std::string & s) { s = readString(in); return in; @@ -450,11 +450,11 @@ Error readError(Source & source) void StringSink::operator () (std::string_view data) { static bool warned = false; - if (!warned && s->size() > threshold) { + if (!warned && s.size() > threshold) { warnLargeDump(); warned = true; } - s->append(data); + s.append(data); } size_t ChainSource::read(char * data, size_t len) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 0fe6e8332..13da26c6a 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -154,12 +154,13 @@ private: /* A sink that writes data to a string. */ struct StringSink : Sink { - ref<std::string> s; - StringSink() : s(make_ref<std::string>()) { }; - explicit StringSink(const size_t reservedSize) : s(make_ref<std::string>()) { - s->reserve(reservedSize); + std::string s; + StringSink() { } + explicit StringSink(const size_t reservedSize) + { + s.reserve(reservedSize); }; - StringSink(ref<std::string> s) : s(s) { }; + StringSink(std::string && s) : s(std::move(s)) { }; void operator () (std::string_view data) override; }; @@ -167,9 +168,9 @@ struct StringSink : Sink /* A source that reads data from a string. */ struct StringSource : Source { - const string & s; + std::string_view s; size_t pos; - StringSource(const string & _s) : s(_s), pos(0) { } + StringSource(std::string_view s) : s(s), pos(0) { } size_t read(char * data, size_t len) override; }; @@ -317,10 +318,10 @@ inline Sink & operator << (Sink & sink, uint64_t n) return sink; } -Sink & operator << (Sink & sink, const string & s); +Sink & operator << (Sink & in, const Error & ex); +Sink & operator << (Sink & sink, std::string_view s); Sink & operator << (Sink & sink, const Strings & s); Sink & operator << (Sink & sink, const StringSet & s); -Sink & operator << (Sink & in, const Error & ex); MakeError(SerialisationError, Error); @@ -363,10 +364,10 @@ inline uint64_t readLongLong(Source & source) void readPadding(size_t len, Source & source); size_t readString(char * buf, size_t max, Source & source); -string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); +std::string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); template<class T> T readStrings(Source & source); -Source & operator >> (Source & in, string & s); +Source & operator >> (Source & in, std::string & s); template<typename T> Source & operator >> (Source & in, T & n) diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc new file mode 100644 index 000000000..9510a5f0c --- /dev/null +++ b/src/libutil/suggestions.cc @@ -0,0 +1,114 @@ +#include "suggestions.hh" +#include "ansicolor.hh" +#include "util.hh" +#include <algorithm> + +namespace nix { + +int levenshteinDistance(std::string_view first, std::string_view second) +{ + // Implementation borrowed from + // https://en.wikipedia.org/wiki/Levenshtein_distance#Iterative_with_two_matrix_rows + + int m = first.size(); + int n = second.size(); + + auto v0 = std::vector<int>(n+1); + auto v1 = std::vector<int>(n+1); + + for (auto i = 0; i <= n; i++) + v0[i] = i; + + for (auto i = 0; i < m; i++) { + v1[0] = i+1; + + for (auto j = 0; j < n; j++) { + auto deletionCost = v0[j+1] + 1; + auto insertionCost = v1[j] + 1; + auto substitutionCost = first[i] == second[j] ? v0[j] : v0[j] + 1; + v1[j+1] = std::min({deletionCost, insertionCost, substitutionCost}); + } + + std::swap(v0, v1); + } + + return v0[n]; +} + +Suggestions Suggestions::bestMatches ( + std::set<std::string> allMatches, + std::string query) +{ + std::set<Suggestion> res; + for (const auto & possibleMatch : allMatches) { + res.insert(Suggestion { + .distance = levenshteinDistance(query, possibleMatch), + .suggestion = possibleMatch, + }); + } + return Suggestions { res }; +} + +Suggestions Suggestions::trim(int limit, int maxDistance) const +{ + std::set<Suggestion> res; + + int count = 0; + + for (auto & elt : suggestions) { + if (count >= limit || elt.distance > maxDistance) + break; + count++; + res.insert(elt); + } + + return Suggestions{res}; +} + +std::string Suggestion::to_string() const +{ + return ANSI_WARNING + filterANSIEscapes(suggestion) + ANSI_NORMAL; +} + +std::string Suggestions::to_string() const +{ + switch (suggestions.size()) { + case 0: + return ""; + case 1: + return suggestions.begin()->to_string(); + default: { + std::string res = "one of "; + auto iter = suggestions.begin(); + res += iter->to_string(); // Iter can’t be end() because the container isn’t null + iter++; + auto last = suggestions.end(); last--; + for ( ; iter != suggestions.end() ; iter++) { + res += (iter == last) ? " or " : ", "; + res += iter->to_string(); + } + return res; + } + } +} + +Suggestions & Suggestions::operator+=(const Suggestions & other) +{ + suggestions.insert( + other.suggestions.begin(), + other.suggestions.end() + ); + return *this; +} + +std::ostream & operator<<(std::ostream & str, const Suggestion & suggestion) +{ + return str << suggestion.to_string(); +} + +std::ostream & operator<<(std::ostream & str, const Suggestions & suggestions) +{ + return str << suggestions.to_string(); +} + +} diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh new file mode 100644 index 000000000..d54dd8e31 --- /dev/null +++ b/src/libutil/suggestions.hh @@ -0,0 +1,102 @@ +#pragma once + +#include "comparator.hh" +#include "types.hh" +#include <set> + +namespace nix { + +int levenshteinDistance(std::string_view first, std::string_view second); + +/** + * A potential suggestion for the cli interface. + */ +class Suggestion { +public: + int distance; // The smaller the better + std::string suggestion; + + std::string to_string() const; + + GENERATE_CMP(Suggestion, me->distance, me->suggestion) +}; + +class Suggestions { +public: + std::set<Suggestion> suggestions; + + std::string to_string() const; + + Suggestions trim( + int limit = 5, + int maxDistance = 2 + ) const; + + static Suggestions bestMatches ( + std::set<std::string> allMatches, + std::string query + ); + + Suggestions& operator+=(const Suggestions & other); +}; + +std::ostream & operator<<(std::ostream & str, const Suggestion &); +std::ostream & operator<<(std::ostream & str, const Suggestions &); + +// Either a value of type `T`, or some suggestions +template<typename T> +class OrSuggestions { +public: + using Raw = std::variant<T, Suggestions>; + + Raw raw; + + T* operator ->() + { + return &**this; + } + + T& operator *() + { + return std::get<T>(raw); + } + + operator bool() const noexcept + { + return std::holds_alternative<T>(raw); + } + + OrSuggestions(T t) + : raw(t) + { + } + + OrSuggestions() + : raw(Suggestions{}) + { + } + + static OrSuggestions<T> failed(const Suggestions & s) + { + auto res = OrSuggestions<T>(); + res.raw = s; + return res; + } + + static OrSuggestions<T> failed() + { + return OrSuggestions<T>::failed(Suggestions{}); + } + + const Suggestions & getSuggestions() + { + static Suggestions noSuggestions; + if (const auto & suggestions = std::get_if<Suggestions>(&raw)) + return *suggestions; + else + return noSuggestions; + } + +}; + +} diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 50e691a3d..790bc943a 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -93,9 +93,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_set_pathname(entry, + archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + // Patch hardlink path + const char *original_hardlink = archive_entry_hardlink(entry); + if (original_hardlink) { + archive_entry_copy_hardlink(entry, + (destDir + "/" + original_hardlink).c_str()); + } + archive.check(archive_read_extract(archive.archive, entry, flags)); } diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc index 2efa3266b..bbbf3500f 100644 --- a/src/libutil/tests/compression.cc +++ b/src/libutil/tests/compression.cc @@ -12,17 +12,17 @@ namespace nix { } TEST(compress, noneMethodDoesNothingToTheInput) { - ref<std::string> o = compress("none", "this-is-a-test"); + auto o = compress("none", "this-is-a-test"); - ASSERT_EQ(*o, "this-is-a-test"); + ASSERT_EQ(o, "this-is-a-test"); } TEST(decompress, decompressNoneCompressed) { auto method = "none"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - ref<std::string> o = decompress(method, str); + auto o = decompress(method, str); - ASSERT_EQ(*o, str); + ASSERT_EQ(o, str); } TEST(decompress, decompressEmptyCompressed) { @@ -30,33 +30,33 @@ namespace nix { // (Content-Encoding == ""). auto method = ""; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - ref<std::string> o = decompress(method, str); + auto o = decompress(method, str); - ASSERT_EQ(*o, str); + ASSERT_EQ(o, str); } TEST(decompress, decompressXzCompressed) { auto method = "xz"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - ref<std::string> o = decompress(method, *compress(method, str)); + auto o = decompress(method, compress(method, str)); - ASSERT_EQ(*o, str); + ASSERT_EQ(o, str); } TEST(decompress, decompressBzip2Compressed) { auto method = "bzip2"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - ref<std::string> o = decompress(method, *compress(method, str)); + auto o = decompress(method, compress(method, str)); - ASSERT_EQ(*o, str); + ASSERT_EQ(o, str); } TEST(decompress, decompressBrCompressed) { auto method = "br"; auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf"; - ref<std::string> o = decompress(method, *compress(method, str)); + auto o = decompress(method, compress(method, str)); - ASSERT_EQ(*o, str); + ASSERT_EQ(o, str); } TEST(decompress, decompressInvalidInputThrowsCompressionError) { @@ -77,7 +77,7 @@ namespace nix { (*sink)(inputString); sink->finish(); - ASSERT_STREQ((*strSink.s).c_str(), inputString); + ASSERT_STREQ(strSink.s.c_str(), inputString); } TEST(makeCompressionSink, compressAndDecompress) { @@ -90,7 +90,7 @@ namespace nix { sink->finish(); decompressionSink->finish(); - ASSERT_STREQ((*strSink.s).c_str(), inputString); + ASSERT_STREQ(strSink.s.c_str(), inputString); } } diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 0ebdaf3db..8be6730dd 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -161,7 +161,7 @@ namespace nix { Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#"); + ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); } TEST(Config, setSettingAlias) { diff --git a/src/libutil/tests/fmt.cc b/src/libutil/tests/fmt.cc new file mode 100644 index 000000000..33772162c --- /dev/null +++ b/src/libutil/tests/fmt.cc @@ -0,0 +1,68 @@ +#include "fmt.hh" + +#include <gtest/gtest.h> + +#include <regex> + +namespace nix { +/* ----------- tests for fmt.hh -------------------------------------------------*/ + + TEST(hiliteMatches, noHighlight) { + ASSERT_STREQ(hiliteMatches("Hello, world!", std::vector<std::smatch>(), "(", ")").c_str(), "Hello, world!"); + } + + TEST(hiliteMatches, simpleHighlight) { + std::string str = "Hello, world!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world)!" + ); + } + + TEST(hiliteMatches, multipleMatches) { + std::string str = "Hello, world, world, world, world, world, world, Hello!"; + std::regex re = std::regex("world"); + auto matches = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "Hello, (world), (world), (world), (world), (world), (world), Hello!" + ); + } + + TEST(hiliteMatches, overlappingMatches) { + std::string str = "world, Hello, world, Hello, world, Hello, world, Hello, world!"; + std::regex re = std::regex("Hello, world"); + std::regex re2 = std::regex("world, Hello"); + auto v = std::vector(std::sregex_iterator(str.begin(), str.end(), re), std::sregex_iterator()); + for(auto it = std::sregex_iterator(str.begin(), str.end(), re2); it != std::sregex_iterator(); ++it) { + v.push_back(*it); + } + ASSERT_STREQ( + hiliteMatches(str, v, "(", ")").c_str(), + "(world, Hello, world, Hello, world, Hello, world, Hello, world)!" + ); + } + + TEST(hiliteMatches, complexOverlappingMatches) { + std::string str = "legacyPackages.x86_64-linux.git-crypt"; + std::vector regexes = { + std::regex("t-cry"), + std::regex("ux\\.git-cry"), + std::regex("git-c"), + std::regex("pt"), + }; + std::vector<std::smatch> matches; + for(auto regex : regexes) + { + for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { + matches.push_back(*it); + } + } + ASSERT_STREQ( + hiliteMatches(str, matches, "(", ")").c_str(), + "legacyPackages.x86_64-lin(ux.git-crypt)" + ); + } +} diff --git a/src/libutil/tests/suggestions.cc b/src/libutil/tests/suggestions.cc new file mode 100644 index 000000000..279994abc --- /dev/null +++ b/src/libutil/tests/suggestions.cc @@ -0,0 +1,43 @@ +#include "suggestions.hh" +#include <gtest/gtest.h> + +namespace nix { + + struct LevenshteinDistanceParam { + std::string s1, s2; + int distance; + }; + + class LevenshteinDistanceTest : + public testing::TestWithParam<LevenshteinDistanceParam> { + }; + + TEST_P(LevenshteinDistanceTest, CorrectlyComputed) { + auto params = GetParam(); + + ASSERT_EQ(levenshteinDistance(params.s1, params.s2), params.distance); + ASSERT_EQ(levenshteinDistance(params.s2, params.s1), params.distance); + } + + INSTANTIATE_TEST_SUITE_P(LevenshteinDistance, LevenshteinDistanceTest, + testing::Values( + LevenshteinDistanceParam{"foo", "foo", 0}, + LevenshteinDistanceParam{"foo", "", 3}, + LevenshteinDistanceParam{"", "", 0}, + LevenshteinDistanceParam{"foo", "fo", 1}, + LevenshteinDistanceParam{"foo", "oo", 1}, + LevenshteinDistanceParam{"foo", "fao", 1}, + LevenshteinDistanceParam{"foo", "abc", 3} + ) + ); + + TEST(Suggestions, Trim) { + auto suggestions = Suggestions::bestMatches({"foooo", "bar", "fo", "gao"}, "foo"); + auto onlyOne = suggestions.trim(1); + ASSERT_EQ(onlyOne.suggestions.size(), 1); + ASSERT_TRUE(onlyOne.suggestions.begin()->suggestion == "fo"); + + auto closest = suggestions.trim(999, 2); + ASSERT_EQ(closest.suggestions.size(), 3); + } +} diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 58df9c5ac..92972ed14 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -4,6 +4,8 @@ #include <limits.h> #include <gtest/gtest.h> +#include <numeric> + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ @@ -282,6 +284,17 @@ namespace nix { ASSERT_EQ(decoded, s); } + TEST(base64Encode, encodeAndDecodeNonPrintable) { + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); + } + /* ---------------------------------------------------------------------------- * base64Decode * --------------------------------------------------------------------------*/ @@ -294,6 +307,10 @@ namespace nix { ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); } + TEST(base64Decode, decodeThrowsOnInvalidChar) { + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); + } + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index aff58e9ee..f20e2dc41 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -5,9 +5,9 @@ namespace nix { /* ----------- tests for url.hh --------------------------------------------------*/ - string print_map(std::map<string, string> m) { - std::map<string, string>::iterator it; - string s = "{ "; + std::string print_map(std::map<std::string, std::string> m) { + std::map<std::string, std::string>::iterator it; + std::string s = "{ "; for (it = m.begin(); it != m.end(); ++it) { s += "{ "; s += it->first; @@ -262,21 +262,21 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(percentDecode, decodesUrlEncodedString) { - string s = "==@=="; - string d = percentDecode("%3D%3D%40%3D%3D"); + std::string s = "==@=="; + std::string d = percentDecode("%3D%3D%40%3D%3D"); ASSERT_EQ(d, s); } TEST(percentDecode, multipleDecodesAreIdempotent) { - string once = percentDecode("%3D%3D%40%3D%3D"); - string twice = percentDecode(once); + std::string once = percentDecode("%3D%3D%40%3D%3D"); + std::string twice = percentDecode(once); ASSERT_EQ(once, twice); } TEST(percentDecode, trailingPercent) { - string s = "==@==%"; - string d = percentDecode("%3D%3D%40%3D%3D%25"); + std::string s = "==@==%"; + std::string d = percentDecode("%3D%3D%40%3D%3D%25"); ASSERT_EQ(d, s); } diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 857ee91f8..dc4067f1b 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,13 +1,10 @@ #include "thread-pool.hh" -#include "affinity.hh" namespace nix { ThreadPool::ThreadPool(size_t _maxThreads) : maxThreads(_maxThreads) { - restoreAffinity(); // FIXME - if (!maxThreads) { maxThreads = std::thread::hardware_concurrency(); if (!maxThreads) maxThreads = 1; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 9c85fef62..00ba567c6 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -6,26 +6,23 @@ #include <set> #include <string> #include <map> +#include <variant> #include <vector> namespace nix { -using std::list; -using std::set; -using std::vector; -using std::string; - -typedef list<string> Strings; -typedef set<string> StringSet; -typedef std::map<string, string> StringMap; +typedef std::list<std::string> Strings; +typedef std::set<std::string> StringSet; +typedef std::map<std::string, std::string> StringMap; +typedef std::map<std::string, std::string> StringPairs; /* Paths are just strings. */ +typedef std::string Path; +typedef std::string_view PathView; +typedef std::list<Path> Paths; +typedef std::set<Path> PathSet; -typedef string Path; -typedef list<Path> Paths; -typedef set<Path> PathSet; - -typedef vector<std::pair<string, string>> Headers; +typedef std::vector<std::pair<std::string, std::string>> Headers; /* Helper class to run code at startup. */ template<typename T> @@ -46,4 +43,63 @@ struct Explicit { } }; + +/* This wants to be a little bit like rust's Cow type. + Some parts of the evaluator benefit greatly from being able to reuse + existing allocations for strings, but have to be able to also use + newly allocated storage for values. + + We do not define implicit conversions, even with ref qualifiers, + since those can easily become ambiguous to the reader and can degrade + into copying behaviour we want to avoid. */ +class BackedStringView { +private: + std::variant<std::string, std::string_view> data; + + /* Needed to introduce a temporary since operator-> must return + a pointer. Without this we'd need to store the view object + even when we already own a string. */ + class Ptr { + private: + std::string_view view; + public: + Ptr(std::string_view view): view(view) {} + const std::string_view * operator->() const { return &view; } + }; + +public: + BackedStringView(std::string && s): data(std::move(s)) {} + BackedStringView(std::string_view sv): data(sv) {} + template<size_t N> + BackedStringView(const char (& lit)[N]): data(std::string_view(lit)) {} + + BackedStringView(const BackedStringView &) = delete; + BackedStringView & operator=(const BackedStringView &) = delete; + + /* We only want move operations defined since the sole purpose of + this type is to avoid copies. */ + BackedStringView(BackedStringView && other) = default; + BackedStringView & operator=(BackedStringView && other) = default; + + bool isOwned() const + { + return std::holds_alternative<std::string>(data); + } + + std::string toOwned() && + { + return isOwned() + ? std::move(std::get<std::string>(data)) + : std::string(std::get<std::string_view>(data)); + } + + std::string_view operator*() const + { + return isOwned() + ? std::get<std::string>(data) + : std::get<std::string_view>(data); + } + Ptr operator->() const { return Ptr(**this); } +}; + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index bc841f425..b833038a9 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,4 @@ #include "util.hh" -#include "affinity.hh" #include "sync.hh" #include "finally.hh" #include "serialise.hh" @@ -82,7 +81,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv) } -Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks) +Path absPath(Path path, std::optional<PathView> dir, bool resolveSymlinks) { if (path[0] != '/') { if (!dir) { @@ -96,28 +95,28 @@ Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks) if (!getcwd(buf, sizeof(buf))) #endif throw SysError("cannot get cwd"); - dir = buf; + path = concatStrings(buf, "/", path); #ifdef __GNU__ free(buf); #endif - } - path = *dir + "/" + path; + } else + path = concatStrings(*dir, "/", path); } return canonPath(path, resolveSymlinks); } -Path canonPath(const Path & path, bool resolveSymlinks) +Path canonPath(PathView path, bool resolveSymlinks) { assert(path != ""); - string s; + std::string s; + s.reserve(256); if (path[0] != '/') throw Error("not an absolute path: '%1%'", path); - string::const_iterator i = path.begin(), end = path.end(); - string temp; + std::string temp; /* Count the number of times we follow a symlink and stop at some arbitrary (but high) limit to prevent infinite loops. */ @@ -126,33 +125,37 @@ Path canonPath(const Path & path, bool resolveSymlinks) while (1) { /* Skip slashes. */ - while (i != end && *i == '/') i++; - if (i == end) break; + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; /* Ignore `.'. */ - if (*i == '.' && (i + 1 == end || i[1] == '/')) - i++; + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); /* If `..', delete the last component. */ - else if (*i == '.' && i + 1 < end && i[1] == '.' && - (i + 2 == end || i[2] == '/')) + else if (path == ".." || path.substr(0, 3) == "../") { if (!s.empty()) s.erase(s.rfind('/')); - i += 2; + path.remove_prefix(2); } /* Normal component; copy it. */ else { s += '/'; - while (i != end && *i != '/') s += *i++; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } /* If s points to a symlink, resolve it and continue from there */ if (resolveSymlinks && isLink(s)) { if (++followCount >= maxFollow) throw Error("infinite symlink recursion in path '%1%'", path); - temp = readLink(s) + string(i, end); - i = temp.begin(); - end = temp.end(); + temp = concatStrings(readLink(s), path); + path = temp; if (!temp.empty() && temp[0] == '/') { s.clear(); /* restart for symlinks pointing to absolute path */ } else { @@ -165,14 +168,14 @@ Path canonPath(const Path & path, bool resolveSymlinks) } } - return s.empty() ? "/" : s; + return s.empty() ? "/" : std::move(s); } -Path dirOf(const Path & path) +Path dirOf(const PathView path) { Path::size_type pos = path.rfind('/'); - if (pos == string::npos) + if (pos == std::string::npos) return "."; return pos == 0 ? "/" : Path(path, 0, pos); } @@ -188,7 +191,7 @@ std::string_view baseNameOf(std::string_view path) last -= 1; auto pos = path.rfind('/', last); - if (pos == string::npos) + if (pos == std::string::npos) pos = 0; else pos += 1; @@ -197,16 +200,16 @@ std::string_view baseNameOf(std::string_view path) } -bool isInDir(const Path & path, const Path & dir) +bool isInDir(std::string_view path, std::string_view dir) { - return path[0] == '/' - && string(path, 0, dir.size()) == dir + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir && path.size() >= dir.size() + 2 && path[dir.size()] == '/'; } -bool isDirOrInDir(const Path & path, const Path & dir) +bool isDirOrInDir(std::string_view path, std::string_view dir) { return path == dir || isInDir(path, dir); } @@ -246,7 +249,7 @@ Path readLink(const Path & path) else throw SysError("reading symbolic link '%1%'", path); else if (rlSize < bufSize) - return string(buf.data(), rlSize); + return std::string(buf.data(), rlSize); } } @@ -266,7 +269,7 @@ DirEntries readDirectory(DIR *dir, const Path & path) struct dirent * dirent; while (errno = 0, dirent = readdir(dir)) { /* sic */ checkInterrupt(); - string name = dirent->d_name; + std::string name = dirent->d_name; if (name == "." || name == "..") continue; entries.emplace_back(name, dirent->d_ino, #ifdef HAVE_STRUCT_DIRENT_D_TYPE @@ -300,7 +303,7 @@ unsigned char getFileType(const Path & path) } -string readFile(int fd) +std::string readFile(int fd) { struct stat st; if (fstat(fd, &st) == -1) @@ -310,7 +313,7 @@ string readFile(int fd) } -string readFile(const Path & path) +std::string readFile(const Path & path) { AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); if (!fd) @@ -363,9 +366,9 @@ void writeFile(const Path & path, Source & source, mode_t mode) } } -string readLine(int fd) +std::string readLine(int fd) { - string s; + std::string s; while (1) { checkInterrupt(); char ch; @@ -384,7 +387,7 @@ string readLine(int fd) } -void writeLine(int fd, string s) +void writeLine(int fd, std::string s) { s += '\n'; writeFull(fd, s); @@ -395,7 +398,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) { checkInterrupt(); - string name(baseNameOf(path)); + std::string name(baseNameOf(path)); struct stat st; if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { @@ -512,6 +515,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); return {std::move(fd), tmpl}; } @@ -562,8 +566,8 @@ Path getConfigDir() std::vector<Path> getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); - std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector<Path> result = tokenizeString<std::vector<std::string>>(configDirs, ":"); result.insert(result.begin(), configHome); return result; } @@ -666,11 +670,13 @@ void writeFull(int fd, std::string_view s, bool allowInterrupts) } -string drainFD(int fd, bool block, const size_t reserveSize) +std::string drainFD(int fd, bool block, const size_t reserveSize) { - StringSink sink(reserveSize); + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); drainFD(fd, sink, block); - return std::move(*sink.s); + return std::move(sink.s); } @@ -713,7 +719,7 @@ void drainFD(int fd, Sink & sink, bool block) AutoDelete::AutoDelete() : del{false} {} -AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) { del = true; this->recursive = recursive; @@ -903,7 +909,7 @@ int Pid::wait() return status; } if (errno != EINTR) - throw SysError("cannot get child exit status"); + throw SysError("cannot get exit status of PID %d", pid); checkInterrupt(); } } @@ -939,9 +945,6 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - ProcessOptions options; - options.allowVfork = false; - Pid pid = startProcess([&]() { if (setuid(uid) == -1) @@ -964,7 +967,7 @@ void killUser(uid_t uid) } _exit(0); - }, options); + }); int status = pid.wait(); if (status != 0) @@ -1006,7 +1009,6 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) throw SysError("setting death signal"); #endif - restoreAffinity(); fun(); } catch (std::exception & e) { try { @@ -1034,7 +1036,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) return res; } -string runProgram(Path program, bool searchPath, const Strings & args, +std::string runProgram(Path program, bool searchPath, const Strings & args, const std::optional<std::string> & input) { auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); @@ -1059,7 +1061,7 @@ std::pair<int, std::string> runProgram(RunOptions && options) status = e.status; } - return {status, std::move(*sink.s)}; + return {status, std::move(sink.s)}; } void runProgram2(const RunOptions & options) @@ -1085,8 +1087,7 @@ void runProgram2(const RunOptions & options) // vfork implies that the environment of the main process and the fork will // be shared (technically this is undefined, but in practice that's the // case), so we can't use it if we alter the environment - if (options.environment) - processOptions.allowVfork = false; + processOptions.allowVfork = !options.environment; /* Fork. */ Pid pid = startProcess([&]() { @@ -1173,7 +1174,7 @@ void runProgram2(const RunOptions & options) } -void closeMostFDs(const set<int> & exceptions) +void closeMostFDs(const std::set<int> & exceptions) { #if __linux__ try { @@ -1209,7 +1210,7 @@ void closeOnExec(int fd) ////////////////////////////////////////////////////////////////////// -bool _isInterrupted = false; +std::atomic<bool> _isInterrupted = false; static thread_local bool interruptThrown = false; thread_local std::function<bool()> interruptCheck; @@ -1234,45 +1235,45 @@ void _interrupted() ////////////////////////////////////////////////////////////////////// -template<class C> C tokenizeString(std::string_view s, const string & separators) +template<class C> C tokenizeString(std::string_view s, std::string_view separators) { C result; - string::size_type pos = s.find_first_not_of(separators, 0); - while (pos != string::npos) { - string::size_type end = s.find_first_of(separators, pos + 1); - if (end == string::npos) end = s.size(); - string token(s, pos, end - pos); - result.insert(result.end(), token); + auto pos = s.find_first_not_of(separators, 0); + while (pos != std::string::npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == std::string::npos) end = s.size(); + result.insert(result.end(), std::string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } return result; } -template Strings tokenizeString(std::string_view s, const string & separators); -template StringSet tokenizeString(std::string_view s, const string & separators); -template vector<string> tokenizeString(std::string_view s, const string & separators); +template Strings tokenizeString(std::string_view s, std::string_view separators); +template StringSet tokenizeString(std::string_view s, std::string_view separators); +template std::vector<std::string> tokenizeString(std::string_view s, std::string_view separators); -string chomp(std::string_view s) +std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); - return i == string::npos ? "" : string(s, 0, i + 1); + return i == std::string_view::npos ? "" : std::string(s, 0, i + 1); } -string trim(const string & s, const string & whitespace) +std::string trim(std::string_view s, std::string_view whitespace) { auto i = s.find_first_not_of(whitespace); - if (i == string::npos) return ""; + if (i == s.npos) return ""; auto j = s.find_last_not_of(whitespace); - return string(s, i, j == string::npos ? j : j - i + 1); + return std::string(s, i, j == s.npos ? j : j - i + 1); } -string replaceStrings(std::string_view s, - const std::string & from, const std::string & to) +std::string replaceStrings( + std::string res, + std::string_view from, + std::string_view to) { - string res(s); if (from.empty()) return res; size_t pos = 0; while ((pos = res.find(from, pos)) != std::string::npos) { @@ -1283,20 +1284,19 @@ string replaceStrings(std::string_view s, } -std::string rewriteStrings(const std::string & _s, const StringMap & rewrites) +std::string rewriteStrings(std::string s, const StringMap & rewrites) { - auto s = _s; for (auto & i : rewrites) { if (i.first == i.second) continue; size_t j = 0; - while ((j = s.find(i.first, j)) != string::npos) + while ((j = s.find(i.first, j)) != std::string::npos) s.replace(j, i.first.size(), i.second); } return s; } -string statusToString(int status) +std::string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { if (WIFEXITED(status)) @@ -1344,9 +1344,11 @@ std::string toLower(const std::string & s) } -std::string shellEscape(const std::string & s) +std::string shellEscape(const std::string_view s) { - std::string r = "'"; + std::string r; + r.reserve(s.size() + 2); + r += "'"; for (auto & i : s) if (i == '\'') r += "'\\''"; else r += i; r += '\''; @@ -1356,11 +1358,15 @@ std::string shellEscape(const std::string & s) void ignoreException() { + /* Make sure no exceptions leave this function. + printError() also throws when remote is closed. */ try { - throw; - } catch (std::exception & e) { - printError("error (ignored): %1%", e.what()); - } + try { + throw; + } catch (std::exception & e) { + printError("error (ignored): %1%", e.what()); + } + } catch (...) { } } bool shouldANSI() @@ -1406,7 +1412,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in } } - else if (*i == '\r') + else if (*i == '\r' || *i == '\a') // do nothing for now i++; @@ -1440,12 +1446,11 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in } -static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static std::array<char, 256> base64DecodeChars; +constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -string base64Encode(std::string_view s) +std::string base64Encode(std::string_view s) { - string res; + std::string res; int data = 0, nbits = 0; for (char c : s) { @@ -1464,16 +1469,19 @@ string base64Encode(std::string_view s) } -string base64Decode(std::string_view s) +std::string base64Decode(std::string_view s) { - static std::once_flag flag; - std::call_once(flag, [](){ - base64DecodeChars = { (char)-1 }; + constexpr char npos = -1; + constexpr std::array<char, 256> base64DecodeChars = [&]() { + std::array<char, 256> result{}; + for (auto& c : result) + c = npos; for (int i = 0; i < 64; i++) - base64DecodeChars[(int) base64Chars[i]] = i; - }); + result[base64Chars[i]] = i; + return result; + }(); - string res; + std::string res; unsigned int d = 0, bits = 0; for (char c : s) { @@ -1481,7 +1489,7 @@ string base64Decode(std::string_view s) if (c == '\n') continue; char digit = base64DecodeChars[(unsigned char) c]; - if (digit == -1) + if (digit == npos) throw Error("invalid character in Base64 string: '%c'", c); bits += 6; @@ -1557,7 +1565,22 @@ std::pair<unsigned short, unsigned short> getWindowSize() } -static Sync<std::list<std::function<void()>>> _interruptCallbacks; +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map<Token, std::function<void()>> callbacks; +}; + +static Sync<InterruptCallbacks> _interruptCallbacks; static void signalHandlerThread(sigset_t set) { @@ -1579,8 +1602,19 @@ void triggerInterrupt() _isInterrupted = true; { - auto interruptCallbacks(_interruptCallbacks.lock()); - for (auto & callback : *interruptCallbacks) { + InterruptCallbacks::Token i = 0; + while (true) { + std::function<void()> callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + try { callback(); } catch (...) { @@ -1634,11 +1668,47 @@ void setStackSize(size_t stackSize) #endif } -void restoreProcessContext() +static AutoCloseFD fdSavedMountNamespace; + +void saveMountNamespace() { - restoreSignals(); +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); + if (!fd) + throw SysError("saving parent mount namespace"); + fdSavedMountNamespace = std::move(fd); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + } catch (Error & e) { + debug(e.msg()); + } +#endif +} - restoreAffinity(); +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } #if __linux__ if (savedStackSize) { @@ -1654,27 +1724,28 @@ void restoreProcessContext() /* RAII helper to automatically deregister a callback. */ struct InterruptCallbackImpl : InterruptCallback { - std::list<std::function<void()>>::iterator it; + InterruptCallbacks::Token token; ~InterruptCallbackImpl() override { - _interruptCallbacks.lock()->erase(it); + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); } }; std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> callback) { auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->push_back(callback); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); auto res = std::make_unique<InterruptCallbackImpl>(); - res->it = interruptCallbacks->end(); - res->it--; + res->token = token; return std::unique_ptr<InterruptCallback>(res.release()); } -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +AutoCloseFD createUnixDomainSocket() { AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM #ifdef SOCK_CLOEXEC @@ -1683,19 +1754,16 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) , 0); if (!fdSocket) throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); + return fdSocket; +} - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (path.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", path); - strcpy(addr.sun_path, path.c_str()); - unlink(path.c_str()); +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); - if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%1%'", path); + bind(fdSocket.get(), path); if (chmod(path.c_str(), mode) == -1) throw SysError("changing permissions on '%1%'", path); @@ -1707,7 +1775,67 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) } -string showBytes(uint64_t bytes) +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + + +std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } @@ -1718,8 +1846,8 @@ void commonChildInit(Pipe & logPipe) { logger = makeSimpleLogger(); - const static string pathNullDevice = "/dev/null"; - restoreProcessContext(); + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bee77b53f..20591952d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,9 @@ #include <unistd.h> #include <signal.h> +#include <boost/lexical_cast.hpp> + +#include <atomic> #include <functional> #include <map> #include <sstream> @@ -46,30 +49,32 @@ void clearEnv(); specified directory, or the current directory otherwise. The path is also canonicalised. */ Path absPath(Path path, - std::optional<Path> dir = {}, + std::optional<PathView> dir = {}, bool resolveSymlinks = false); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink components such that each component of the resulting path is *not* a symbolic link. */ -Path canonPath(const Path & path, bool resolveSymlinks = false); +Path canonPath(PathView path, bool resolveSymlinks = false); /* Return the directory part of the given canonical path, i.e., everything before the final `/'. If the path is the root or an immediate child thereof (e.g., `/foo'), this means `/' is returned.*/ -Path dirOf(const Path & path); +Path dirOf(const PathView path); /* Return the base name of the given canonical path, i.e., everything following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); -/* Check whether 'path' is a descendant of 'dir'. */ -bool isInDir(const Path & path, const Path & dir); +/* Check whether 'path' is a descendant of 'dir'. Both paths must be + canonicalized. */ +bool isInDir(std::string_view path, std::string_view dir); -/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ -bool isDirOrInDir(const Path & path, const Path & dir); +/* Check whether 'path' is equal to 'dir' or a descendant of + 'dir'. Both paths must be canonicalized. */ +bool isDirOrInDir(std::string_view path, std::string_view dir); /* Get status of `path'. */ struct stat lstat(const Path & path); @@ -87,22 +92,22 @@ bool isLink(const Path & path); removed. */ struct DirEntry { - string name; + std::string name; ino_t ino; unsigned char type; // one of DT_* - DirEntry(const string & name, ino_t ino, unsigned char type) - : name(name), ino(ino), type(type) { } + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } }; -typedef vector<DirEntry> DirEntries; +typedef std::vector<DirEntry> DirEntries; DirEntries readDirectory(const Path & path); unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ -string readFile(int fd); -string readFile(const Path & path); +std::string readFile(int fd); +std::string readFile(const Path & path); void readFile(const Path & path, Sink & sink); /* Write a string to a file. */ @@ -111,10 +116,10 @@ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666); void writeFile(const Path & path, Source & source, mode_t mode = 0666); /* Read a line from a file descriptor. */ -string readLine(int fd); +std::string readLine(int fd); /* Write a line to a file descriptor. */ -void writeLine(int fd, string s); +void writeLine(int fd, std::string s); /* Delete a path; i.e., in the case of a directory, it is deleted recursively. It's not an error if the path does not exist. The @@ -143,6 +148,9 @@ Path getDataDir(); /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) { + return createDirs(Path(path)); +} /* Create a symlink. */ void createSymlink(const Path & target, const Path & link, @@ -162,7 +170,7 @@ MakeError(EndOfFile, Error); /* Read a file descriptor until EOF occurs. */ -string drainFD(int fd, bool block = true, const size_t reserveSize=0); +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); void drainFD(int fd, Sink & sink, bool block = true); @@ -182,6 +190,7 @@ public: void cancel(); void reset(const Path & p, bool recursive = true); operator Path() const { return path; } + operator PathView() const { return path; } }; @@ -259,10 +268,10 @@ void killUser(uid_t uid); pid to the caller. */ struct ProcessOptions { - string errorPrefix = "error: "; + std::string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; - bool allowVfork = true; + bool allowVfork = false; }; pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); @@ -270,7 +279,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ -string runProgram(Path program, bool searchPath = false, +std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), const std::optional<std::string> & input = {}); @@ -299,8 +308,21 @@ void setStackSize(size_t stackSize); /* Restore the original inherited Unix process context (such as signal - masks, stack size, CPU affinity). */ -void restoreProcessContext(); + masks, stack size). */ +void restoreProcessContext(bool restoreMounts = true); + +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); + +/* Cause this thread to not share any FS attributes with the main + thread, because this causes setns() in restoreMountNamespace() to + fail. */ +void unshareFilesystem(); class ExecError : public Error @@ -321,7 +343,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss); /* Close all file descriptors except those listed in the given set. Good practice in child processes. */ -void closeMostFDs(const set<int> & exceptions); +void closeMostFDs(const std::set<int> & exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); @@ -329,7 +351,7 @@ void closeOnExec(int fd); /* User interruption. */ -extern bool _isInterrupted; +extern std::atomic<bool> _isInterrupted; extern thread_local std::function<bool()> interruptCheck; @@ -350,15 +372,19 @@ MakeError(FormatError, Error); /* String tokenizer. */ -template<class C> C tokenizeString(std::string_view s, const string & separators = " \t\n\r"); +template<class C> C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); /* Concatenate the given strings with a separator between the elements. */ template<class C> -string concatStringsSep(const string & sep, const C & ss) +std::string concatStringsSep(const std::string_view sep, const C & ss) { - string s; + size_t size = 0; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); for (auto & i : ss) { if (s.size() != 0) s += sep; s += i; @@ -366,6 +392,14 @@ string concatStringsSep(const string & sep, const C & ss) return s; } +template<class ... Parts> +auto concatStrings(Parts && ... parts) + -> std::enable_if_t<(... && std::is_convertible_v<Parts, std::string_view>), std::string> +{ + std::string_view views[sizeof...(parts)] = { parts... }; + return concatStringsSep({}, views); +} + /* Add quotes around a collection of strings. */ template<class C> Strings quoteStrings(const C & c) @@ -379,45 +413,47 @@ template<class C> Strings quoteStrings(const C & c) /* Remove trailing whitespace from a string. FIXME: return std::string_view. */ -string chomp(std::string_view s); +std::string chomp(std::string_view s); /* Remove whitespace from the start and end of a string. */ -string trim(const string & s, const string & whitespace = " \n\r\t"); +std::string trim(std::string_view s, std::string_view whitespace = " \n\r\t"); /* Replace all occurrences of a string inside another string. */ -string replaceStrings(std::string_view s, - const std::string & from, const std::string & to); +std::string replaceStrings( + std::string s, + std::string_view from, + std::string_view to); -std::string rewriteStrings(const std::string & s, const StringMap & rewrites); +std::string rewriteStrings(std::string s, const StringMap & rewrites); /* Convert the exit status of a child as returned by wait() into an error string. */ -string statusToString(int status); +std::string statusToString(int status); bool statusOk(int status); /* Parse a string into an integer. */ template<class N> -std::optional<N> string2Int(const std::string & s) +std::optional<N> string2Int(const std::string_view s) { if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed) return std::nullopt; - std::istringstream str(s); - N n; - str >> n; - if (str && str.get() == EOF) return n; - return std::nullopt; + try { + return boost::lexical_cast<N>(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } } /* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or 'T' denoting a binary unit prefix. */ template<class N> -N string2IntWithUnitPrefix(std::string s) +N string2IntWithUnitPrefix(std::string_view s) { N multiplier = 1; if (!s.empty()) { @@ -428,7 +464,7 @@ N string2IntWithUnitPrefix(std::string s) else if (u == 'G') multiplier = 1ULL << 30; else if (u == 'T') multiplier = 1ULL << 40; else throw UsageError("invalid unit specifier '%1%'", u); - s.resize(s.size() - 1); + s.remove_suffix(1); } } if (auto n = string2Int<N>(s)) @@ -438,13 +474,13 @@ N string2IntWithUnitPrefix(std::string s) /* Parse a string into a float. */ template<class N> -std::optional<N> string2Float(const string & s) +std::optional<N> string2Float(const std::string_view s) { - std::istringstream str(s); - N n; - str >> n; - if (str && str.get() == EOF) return n; - return std::nullopt; + try { + return boost::lexical_cast<N>(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } } @@ -461,7 +497,7 @@ std::string toLower(const std::string & s); /* Escape a string as a shell word. */ -std::string shellEscape(const std::string & s); +std::string shellEscape(const std::string_view s); /* Exception handling in destructors: print an error message, then @@ -491,8 +527,8 @@ std::string filterANSIEscapes(const std::string & s, /* Base64 encoding/decoding. */ -string base64Encode(std::string_view s); -string base64Decode(std::string_view s); +std::string base64Encode(std::string_view s); +std::string base64Decode(std::string_view s); /* Remove common leading whitespace from the lines in the string @@ -511,6 +547,29 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_ } +/* Remove and return the first item from a container. */ +template <class T> +std::optional<typename T::value_type> remove_begin(T & c) +{ + auto i = c.begin(); + if (i == c.end()) return {}; + auto v = std::move(*i); + c.erase(i); + return v; +} + + +/* Remove and return the first item from a container. */ +template <class T> +std::optional<typename T::value_type> pop(T & c) +{ + if (c.empty()) return {}; + auto v = std::move(c.front()); + c.pop(); + return v; +} + + template<typename T> class Callback; @@ -571,9 +630,18 @@ extern PathFilter defaultPathFilter; /* Common initialisation performed in child processes. */ void commonChildInit(Pipe & logPipe); +/* Create a Unix domain socket. */ +AutoCloseFD createUnixDomainSocket(); + /* Create a Unix domain socket in listen mode. */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +/* Bind a Unix domain socket to a path. */ +void bind(int fd, const std::string & path); + +/* Connect to a Unix domain socket. */ +void connect(int fd, const std::string & path); + // A Rust/Python-like enumerate() iterator adapter. // Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. diff --git a/src/libutil/xml-writer.cc b/src/libutil/xml-writer.cc index 68857e34d..7993bee9a 100644 --- a/src/libutil/xml-writer.cc +++ b/src/libutil/xml-writer.cc @@ -31,11 +31,12 @@ void XMLWriter::close() void XMLWriter::indent_(size_t depth) { if (!indent) return; - output << string(depth * 2, ' '); + output << std::string(depth * 2, ' '); } -void XMLWriter::openElement(const string & name, +void XMLWriter::openElement( + std::string_view name, const XMLAttrs & attrs) { assert(!closed); @@ -44,7 +45,7 @@ void XMLWriter::openElement(const string & name, writeAttrs(attrs); output << ">"; if (indent) output << std::endl; - pendingElems.push_back(name); + pendingElems.push_back(std::string(name)); } @@ -59,7 +60,8 @@ void XMLWriter::closeElement() } -void XMLWriter::writeEmptyElement(const string & name, +void XMLWriter::writeEmptyElement( + std::string_view name, const XMLAttrs & attrs) { assert(!closed); diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index b98b44526..4c91adee6 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -8,12 +8,8 @@ namespace nix { -using std::string; -using std::map; -using std::list; - -typedef map<string, string> XMLAttrs; +typedef std::map<std::string, std::string> XMLAttrs; class XMLWriter @@ -25,7 +21,7 @@ private: bool indent; bool closed; - list<string> pendingElems; + std::list<std::string> pendingElems; public: @@ -34,11 +30,11 @@ public: void close(); - void openElement(const string & name, + void openElement(std::string_view name, const XMLAttrs & attrs = XMLAttrs()); void closeElement(); - void writeEmptyElement(const string & name, + void writeEmptyElement(std::string_view name, const XMLAttrs & attrs = XMLAttrs()); private: @@ -53,7 +49,7 @@ class XMLOpenElement private: XMLWriter & writer; public: - XMLOpenElement(XMLWriter & writer, const string & name, + XMLOpenElement(XMLWriter & writer, std::string_view name, const XMLAttrs & attrs = XMLAttrs()) : writer(writer) { |