diff options
Diffstat (limited to 'src')
118 files changed, 11686 insertions, 8943 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 38dbe3e58..279ae62f6 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -17,6 +17,7 @@ #include "store-api.hh" #include "derivations.hh" #include "local-store.hh" +#include "legacy.hh" using namespace nix; using std::cin; @@ -37,11 +38,15 @@ static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); } -int main (int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); +static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} +static int _main(int argc, char * * argv) +{ + { logger = makeJSONLogger(*logger); /* Ensure we don't get any SSH passphrase or host key popups. */ @@ -80,7 +85,7 @@ int main (int argc, char * * argv) if (machines.empty()) { std::cerr << "# decline-permanently\n"; - return; + return 0; } string drvPath; @@ -90,17 +95,18 @@ int main (int argc, char * * argv) try { auto s = readString(source); - if (s != "try") return; - } catch (EndOfFile &) { return; } + if (s != "try") return 0; + } catch (EndOfFile &) { return 0; } auto amWilling = readInt(source); auto neededSystem = readString(source); source >> drvPath; auto requiredFeatures = readStrings<std::set<std::string>>(source); - auto canBuildLocally = amWilling - && ( neededSystem == settings.thisSystem - || settings.extraPlatforms.get().count(neededSystem) > 0); + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); /* Error ignored here, will be caught later */ mkdir(currentLoad.c_str(), 0777); @@ -253,6 +259,8 @@ connected: copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } - return; - }); + return 0; + } } + +static RegisterLegacyCommand s1("build-remote", _main); diff --git a/src/build-remote/local.mk b/src/build-remote/local.mk deleted file mode 100644 index 50b0409d1..000000000 --- a/src/build-remote/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += build-remote - -build-remote_DIR := $(d) - -build-remote_INSTALL_DIR := $(libexecdir)/nix - -build-remote_LIBS = libmain libformat libstore libutil - -build-remote_SOURCES := $(d)/build-remote.cc diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h index 07de010b1..5a00da3b4 100644 --- a/src/cpptoml/cpptoml.h +++ b/src/cpptoml/cpptoml.h @@ -4,8 +4,8 @@ * @date May 2013 */ -#ifndef _CPPTOML_H_ -#define _CPPTOML_H_ +#ifndef CPPTOML_H +#define CPPTOML_H #include <algorithm> #include <cassert> @@ -84,11 +84,12 @@ class option return &value_; } - const T& value_or(const T& alternative) const + template <class U> + T value_or(U&& alternative) const { if (!empty_) return value_; - return alternative; + return static_cast<T>(std::forward<U>(alternative)); } private: @@ -295,13 +296,12 @@ struct valid_value_or_string_convertible }; template <class T> -struct value_traits<T, typename std:: - enable_if<valid_value_or_string_convertible<T>:: - value>::type> +struct value_traits<T, typename std::enable_if< + valid_value_or_string_convertible<T>::value>::type> { - using value_type = typename std:: - conditional<valid_value<typename std::decay<T>::type>::value, - typename std::decay<T>::type, std::string>::type; + using value_type = typename std::conditional< + valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; using type = value<value_type>; @@ -312,12 +312,11 @@ struct value_traits<T, typename std:: }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_floating_point< - typename std::decay<T>::type>::value>:: - type> +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_floating_point<typename std::decay<T>::type>::value>::type> { using value_type = typename std::decay<T>::type; @@ -330,11 +329,11 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_signed<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && !std::is_floating_point<typename std::decay<T>::type>::value + && std::is_signed<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -356,11 +355,10 @@ struct value_traits<T, }; template <class T> -struct value_traits<T, - typename std:: - enable_if<!valid_value_or_string_convertible<T>::value - && std::is_unsigned<typename std::decay<T>:: - type>::value>::type> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>::type>::value>::type> { using value_type = int64_t; @@ -395,10 +393,15 @@ struct array_of_trait<array> template <class T> inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); inline std::shared_ptr<array> make_array(); + +namespace detail +{ template <class T> inline std::shared_ptr<T> make_element(); +} + inline std::shared_ptr<table> make_table(); -inline std::shared_ptr<table_array> make_table_array(); +inline std::shared_ptr<table_array> make_table_array(bool is_inline = false); #if defined(CPPTOML_NO_RTTI) /// Base type used to store underlying data type explicitly if RTTI is disabled @@ -576,7 +579,7 @@ class base : public std::enable_shared_from_this<base> #if defined(CPPTOML_NO_RTTI) base_type type() const { - return type_; + return type_; } protected: @@ -698,7 +701,7 @@ inline std::shared_ptr<value<double>> base::as() if (type() == base_type::INT) { auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); - return make_value<double>(static_cast<double>(v->get()));; + return make_value<double>(static_cast<double>(v->get())); } #else if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) @@ -731,7 +734,8 @@ inline std::shared_ptr<const value<double>> base::as() const { #if defined(CPPTOML_NO_RTTI) if (type() == base_type::FLOAT) - return std::static_pointer_cast<const value<double>>(shared_from_this()); + return std::static_pointer_cast<const value<double>>( + shared_from_this()); if (type() == base_type::INT) { @@ -1027,11 +1031,14 @@ inline std::shared_ptr<array> make_array() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<array> make_element<array>() { return make_array(); } +} // namespace detail /** * Obtains a option<vector<T>>. The option will be empty if the array @@ -1060,7 +1067,7 @@ class table; class table_array : public base { friend class table; - friend std::shared_ptr<table_array> make_table_array(); + friend std::shared_ptr<table_array> make_table_array(bool); public: std::shared_ptr<base> clone() const override; @@ -1152,14 +1159,25 @@ class table_array : public base array_.reserve(n); } + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + private: #if defined(CPPTOML_NO_RTTI) - table_array() : base(base_type::TABLE_ARRAY) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) { // nothing } #else - table_array() + table_array(bool is_inline = false) : is_inline_(is_inline) { // nothing } @@ -1169,26 +1187,30 @@ class table_array : public base table_array& operator=(const table_array& rhs) = delete; std::vector<std::shared_ptr<table>> array_; + const bool is_inline_ = false; }; -inline std::shared_ptr<table_array> make_table_array() +inline std::shared_ptr<table_array> make_table_array(bool is_inline) { struct make_shared_enabler : public table_array { - make_shared_enabler() + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) { // nothing } }; - return std::make_shared<make_shared_enabler>(); + return std::make_shared<make_shared_enabler>(is_inline); } +namespace detail +{ template <> inline std::shared_ptr<table_array> make_element<table_array>() { - return make_table_array(); + return make_table_array(true); } +} // namespace detail // The below are overloads for fetching specific value types out of a value // where special casting behavior (like bounds checking) is desired @@ -1679,11 +1701,14 @@ std::shared_ptr<table> make_table() return std::make_shared<make_shared_enabler>(); } +namespace detail +{ template <> inline std::shared_ptr<table> make_element<table>() { return make_table(); } +} // namespace detail template <class T> std::shared_ptr<base> value<T>::clone() const @@ -1702,7 +1727,7 @@ inline std::shared_ptr<base> array::clone() const inline std::shared_ptr<base> table_array::clone() const { - auto result = make_table_array(); + auto result = make_table_array(is_inline()); result->reserve(array_.size()); for (const auto& ptr : array_) result->array_.push_back(ptr->clone()->as_table()); @@ -1738,6 +1763,11 @@ inline bool is_number(char c) return c >= '0' && c <= '9'; } +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + /** * Helper object for consuming expected characters. */ @@ -1766,6 +1796,13 @@ class consumer [&](char c) { (*this)(c); }); } + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + int eat_digits(int len) { int val = 0; @@ -1830,7 +1867,7 @@ inline std::istream& getline(std::istream& input, std::string& line) line.push_back(static_cast<char>(c)); } } -} +} // namespace detail /** * The parser class. @@ -1914,21 +1951,25 @@ class parser std::string full_table_name; bool inserted = false; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table name"); if (!full_table_name.empty()) - full_table_name += "."; + full_table_name += '.'; full_table_name += part; if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif if (b->is_table()) curr_table = static_cast<table*>(b.get()); else if (b->is_table_array()) @@ -1946,16 +1987,23 @@ class parser curr_table->insert(part, make_table()); curr_table = static_cast<table*>(curr_table->get(part).get()); } - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); if (it == end) throw_parse_exception( "Unterminated table declaration; did you forget a ']'?"); + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + // table already existed if (!inserted) { @@ -1969,8 +2017,9 @@ class parser // since it has already been defined. If there aren't any // values, then it was implicitly created by something like // [a.b] - if (curr_table->empty() || std::any_of(curr_table->begin(), - curr_table->end(), is_value)) + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) { throw_parse_exception("Redefinition of table " + full_table_name); @@ -1989,36 +2038,45 @@ class parser if (it == end || *it == ']') throw_parse_exception("Table array name cannot be empty"); - std::string full_ta_name; - while (it != end && *it != ']') - { - auto part = parse_key(it, end, - [](char c) { return c == '.' || c == ']'; }); + auto key_end = [](char c) { return c == ']'; }; + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { if (part.empty()) throw_parse_exception("Empty component of table array name"); if (!full_ta_name.empty()) - full_ta_name += "."; + full_ta_name += '.'; full_ta_name += part; - consume_whitespace(it, end); - if (it != end && *it == '.') - ++it; - consume_whitespace(it, end); - if (curr_table->contains(part)) { +#if !defined(__PGI) auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif // if this is the end of the table array name, add an - // element to the table array that we just looked up + // element to the table array that we just looked up, + // provided it was not declared inline if (it != end && *it == ']') { if (!b->is_table_array()) + { throw_parse_exception("Key " + full_ta_name + " is not a table array"); + } + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + v->get().push_back(make_table()); curr_table = v->get().back().get(); } @@ -2059,15 +2117,16 @@ class parser = static_cast<table*>(curr_table->get(part).get()); } } - } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); // consume the last "]]" - if (it == end) - throw_parse_exception("Unterminated table array name"); - ++it; - if (it == end) + auto eat = make_consumer(it, end, [this]() { throw_parse_exception("Unterminated table array name"); - ++it; + }); + eat(']'); + eat(']'); consume_whitespace(it, end); eol_or_comment(it, end); @@ -2076,7 +2135,35 @@ class parser void parse_key_value(std::string::iterator& it, std::string::iterator& end, table* curr_table) { - auto key = parse_key(it, end, [](char c) { return c == '='; }); + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast<table*>(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + if (curr_table->contains(key)) throw_parse_exception("Key " + key + " already present"); if (it == end || *it != '=') @@ -2087,18 +2174,57 @@ class parser consume_whitespace(it, end); } - template <class Function> - std::string parse_key(std::string::iterator& it, - const std::string::iterator& end, Function&& fun) + template <class KeyEndFinder, class KeyPartHandler> + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) { consume_whitespace(it, end); - if (*it == '"') + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') { - return parse_quoted_key(it, end); + return string_literal(it, end, *it); } else { - auto bke = std::find_if(it, end, std::forward<Function>(fun)); + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); return parse_bare_key(it, bke); } } @@ -2142,12 +2268,6 @@ class parser return key; } - std::string parse_quoted_key(std::string::iterator& it, - const std::string::iterator& end) - { - return string_literal(it, end, '"'); - } - enum class parse_type { STRING = 1, @@ -2193,7 +2313,7 @@ class parser parse_type determine_value_type(const std::string::iterator& it, const std::string::iterator& end) { - if(it == end) + if (it == end) { throw_parse_exception("Failed to parse value type"); } @@ -2209,7 +2329,11 @@ class parser { return *dtype; } - else if (is_number(*it) || *it == '-' || *it == '+') + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) { return determine_number_type(it, end); } @@ -2235,6 +2359,13 @@ class parser auto check_it = it; if (*check_it == '-' || *check_it == '+') ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + while (check_it != end && is_number(*check_it)) ++check_it; if (check_it != end && *check_it == '.') @@ -2283,57 +2414,56 @@ class parser bool consuming = false; std::shared_ptr<value<std::string>> ret; - auto handle_line - = [&](std::string::iterator& local_it, - std::string::iterator& local_end) { - if (consuming) - { - local_it = std::find_if_not(local_it, local_end, is_ws); - - // whole line is whitespace - if (local_it == local_end) - return; - } - - consuming = false; - - while (local_it != local_end) - { - // handle escaped characters - if (delim == '"' && *local_it == '\\') - { - auto check = local_it; - // check if this is an actual escape sequence or a - // whitespace escaping backslash - ++check; - consume_whitespace(check, local_end); - if (check == local_end) - { - consuming = true; - break; - } - - ss << parse_escape_code(local_it, local_end); - continue; - } - - // if we can end the string - if (std::distance(local_it, local_end) >= 3) - { - auto check = local_it; - // check for """ - if (*check++ == delim && *check++ == delim - && *check++ == delim) - { - local_it = check; - ret = make_value<std::string>(ss.str()); - break; - } - } - - ss << *local_it++; - } - }; + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; // handle the remainder of the current line handle_line(it, end); @@ -2514,17 +2644,13 @@ class parser return value; } - bool is_hex(char c) - { - return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - uint32_t hex_to_digit(char c) { if (is_number(c)) return static_cast<uint32_t>(c - '0'); - return 10 + static_cast<uint32_t>( - c - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + return 10 + + static_cast<uint32_t>(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); } std::shared_ptr<base> parse_number(std::string::iterator& it, @@ -2538,17 +2664,23 @@ class parser ++check_it; }; - eat_sign(); + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; - auto eat_numbers = [&]() { + auto eat_digits = [&](bool (*check_char)(char)) { auto beg = check_it; - while (check_it != end && is_number(*check_it)) + while (check_it != end && check_char(*check_it)) { ++check_it; if (check_it != end && *check_it == '_') { ++check_it; - if (check_it == end || !is_number(*check_it)) + if (check_it == end || !check_char(*check_it)) throw_parse_exception("Malformed number"); } } @@ -2557,15 +2689,63 @@ class parser throw_parse_exception("Malformed number"); }; - auto check_no_leading_zero = [&]() { - if (check_it != end && *check_it == '0' && check_it + 1 != check_end - && check_it[1] != '.') + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') { - throw_parse_exception("Numbers may not have leading zeros"); + eat_hex(); + return parse_int(it, check_it, 16); } - }; + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + eat_sign(); check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits<double>::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits<double>::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + eat_numbers(); if (check_it != end @@ -2604,14 +2784,17 @@ class parser } std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, - const std::string::iterator& end) + const std::string::iterator& end, + int base = 10, + const char* prefix = "") { std::string v{it, end}; + v = prefix + v; v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); it = end; try { - return make_value<int64_t>(std::stoll(v)); + return make_value<int64_t>(std::stoll(v, nullptr, base)); } catch (const std::invalid_argument& ex) { @@ -2674,18 +2857,33 @@ class parser std::string::iterator find_end_of_number(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { + auto ret = std::find_if(it, end, [](char c) { return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' - && c != '-' && c != '+'; + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; } std::string::iterator find_end_of_date(std::string::iterator it, std::string::iterator end) { - return std::find_if(it, end, [](char c) { - return !is_number(c) && c != 'T' && c != 'Z' && c != ':' && c != '-' - && c != '+' && c != '.'; + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; }); } @@ -2754,7 +2952,7 @@ class parser if (it == date_end) return make_value(ldate); - eat('T'); + eat.eat_or('T', ' '); local_datetime ldt; static_cast<local_date&>(ldt) = ldate; @@ -2850,9 +3048,9 @@ class parser auto arr = make_array(); while (it != end && *it != ']') { - auto value = parse_value(it, end); - if (auto v = value->as<Value>()) - arr->get().push_back(value); + auto val = parse_value(it, end); + if (auto v = val->as<Value>()) + arr->get().push_back(val); else throw_parse_exception("Arrays must be homogeneous"); skip_whitespace_and_comments(it, end); @@ -2871,7 +3069,7 @@ class parser std::string::iterator& it, std::string::iterator& end) { - auto arr = make_element<Object>(); + auto arr = detail::make_element<Object>(); while (it != end && *it != ']') { @@ -2881,7 +3079,7 @@ class parser arr->get().push_back(((*this).*fun)(it, end)); skip_whitespace_and_comments(it, end); - if (*it != ',') + if (it == end || *it != ',') break; ++it; @@ -2906,8 +3104,11 @@ class parser throw_parse_exception("Unterminated inline table"); consume_whitespace(it, end); - parse_key_value(it, end, tbl.get()); - consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } } while (*it == ','); if (it == end || *it != '}') @@ -2987,7 +3188,8 @@ class parser if (it[4] != '-' || it[7] != '-') return {}; - if (len >= 19 && it[10] == 'T' && is_time(it + 11, date_end)) + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) { // datetime type auto time_end = find_end_of_time(it + 11, date_end); @@ -3243,7 +3445,7 @@ class toml_writer { res += "\\\\"; } - else if ((const uint32_t)*it <= 0x001f) + else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f)) { res += "\\u"; std::stringstream ss; @@ -3274,12 +3476,21 @@ class toml_writer */ void write(const value<double>& v) { - std::ios::fmtflags flags{stream_.flags()}; - - stream_ << std::showpoint; - write(v.get()); - - stream_.flags(flags); + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits<double>::max_digits10) + << v.get(); + + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); + + stream_ << double_str; + has_naked_endline_ = false; } /** @@ -3287,9 +3498,9 @@ class toml_writer * offset_datetime. */ template <class T> - typename std::enable_if<is_one_of<T, int64_t, local_date, local_time, - local_datetime, - offset_datetime>::value>::type + typename std::enable_if< + is_one_of<T, int64_t, local_date, local_time, local_datetime, + offset_datetime>::value>::type write(const value<T>& v) { write(v.get()); @@ -3453,5 +3664,5 @@ inline std::ostream& operator<<(std::ostream& stream, const array& a) a.accept(writer); return stream; } -} -#endif +} // namespace cpptoml +#endif // CPPTOML_H diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 3e0c78f28..13950ab8d 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -45,9 +45,11 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { - if (isUri(s)) - return getDownloader()->downloadCached(state.store, s, true); - else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); } else diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f41905787..9f4b6b411 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,14 +6,17 @@ #include "globals.hh" #include "eval-inline.hh" #include "download.hh" +#include "json.hh" #include <algorithm> +#include <chrono> #include <cstring> #include <unistd.h> #include <sys/time.h> #include <sys/resource.h> +#include <iostream> +#include <fstream> -#include <sys/time.h> #include <sys/resource.h> #if HAVE_BOEHMGC @@ -23,7 +26,6 @@ #endif - namespace nix { @@ -128,6 +130,16 @@ std::ostream & operator << (std::ostream & str, const Value & v) } +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + string showType(const Value & v) { switch (v.type) { @@ -142,8 +154,10 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tFloat: return "a float"; } @@ -755,6 +769,7 @@ void EvalState::evalFile(const Path & path_, Value & v) void EvalState::resetFileCache() { fileEvalCache.clear(); + fileParseCache.clear(); } @@ -1079,9 +1094,13 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } } - void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { + std::optional<FunctionCallTrace> trace; + if (evalSettings.traceFunctionCalls) { + trace.emplace(pos); + } + forceValue(fun, pos); if (fun.type == tPrimOp || fun.type == tPrimOpApp) { @@ -1723,12 +1742,9 @@ bool EvalState::eqValues(Value & v1, Value & v2) } } - void EvalState::printStats() { bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; - Verbosity v = showStats ? lvlInfo : lvlDebug; - printMsg(v, "evaluation statistics:"); struct rusage buf; getrusage(RUSAGE_SELF, &buf); @@ -1739,62 +1755,107 @@ void EvalState::printStats() uint64_t bValues = nrValues * sizeof(Value); uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); - printMsg(v, format(" time elapsed: %1%") % cpuTime); - printMsg(v, format(" size of a value: %1%") % sizeof(Value)); - printMsg(v, format(" size of an attr: %1%") % sizeof(Attr)); - printMsg(v, format(" environments allocated count: %1%") % nrEnvs); - printMsg(v, format(" environments allocated bytes: %1%") % bEnvs); - printMsg(v, format(" list elements count: %1%") % nrListElems); - printMsg(v, format(" list elements bytes: %1%") % bLists); - printMsg(v, format(" list concatenations: %1%") % nrListConcats); - printMsg(v, format(" values allocated count: %1%") % nrValues); - printMsg(v, format(" values allocated bytes: %1%") % bValues); - printMsg(v, format(" sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets); - printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates); - printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied); - printMsg(v, format(" symbols in symbol table: %1%") % symbols.size()); - printMsg(v, format(" size of symbol table: %1%") % symbols.totalSize()); - printMsg(v, format(" number of thunks: %1%") % nrThunks); - printMsg(v, format(" number of thunks avoided: %1%") % nrAvoided); - printMsg(v, format(" number of attr lookups: %1%") % nrLookups); - printMsg(v, format(" number of primop calls: %1%") % nrPrimOpCalls); - printMsg(v, format(" number of function calls: %1%") % nrFunctionCalls); - printMsg(v, format(" total allocations: %1% bytes") % (bEnvs + bLists + bValues + bAttrsets)); - #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); - printMsg(v, format(" current Boehm heap size: %1% bytes") % heapSize); - printMsg(v, format(" total Boehm heap allocations: %1% bytes") % totalBytes); +#endif + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH","-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime",cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.size()); + syms.attr("bytes", symbols.totalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); +#if HAVE_BOEHMGC + { + auto gc = topObj.object("gc"); + gc.attr("heapSize", heapSize); + gc.attr("totalBytes", totalBytes); + } #endif - if (countCalls) { - v = lvlInfo; - - printMsg(v, format("calls to %1% primops:") % primOpCalls.size()); - typedef std::multimap<size_t, Symbol> PrimOpCalls_; - PrimOpCalls_ primOpCalls_; - for (auto & i : primOpCalls) - primOpCalls_.insert(std::pair<size_t, Symbol>(i.second, i.first)); - for (auto i = primOpCalls_.rbegin(); i != primOpCalls_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second); - - printMsg(v, format("calls to %1% functions:") % functionCalls.size()); - typedef std::multimap<size_t, ExprLambda *> FunctionCalls_; - FunctionCalls_ functionCalls_; - for (auto & i : functionCalls) - functionCalls_.insert(std::pair<size_t, ExprLambda *>(i.second, i.first)); - for (auto i = functionCalls_.rbegin(); i != functionCalls_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second->showNamePos()); - - printMsg(v, format("evaluations of %1% attributes:") % attrSelects.size()); - typedef std::multimap<size_t, Pos> AttrSelects_; - AttrSelects_ attrSelects_; - for (auto & i : attrSelects) - attrSelects_.insert(std::pair<size_t, Pos>(i.second, i.first)); - for (auto i = attrSelects_.rbegin(); i != attrSelects_.rend(); ++i) - printMsg(v, format("%1$10d %2%") % i->first % i->second); + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto & i : primOpCalls) + obj.attr(i.first, i.second); + } + { + auto list = topObj.list("functions"); + for (auto & i : functionCalls) { + auto obj = list.object(); + if (i.first->name.set()) + obj.attr("name", (const string &) i.first->name); + else + obj.attr("name", nullptr); + if (i.first->pos) { + obj.attr("file", (const string &) i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); + } + } + { + auto list = topObj.list("attributes"); + for (auto & i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const string &) i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); + } + } + } + if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string & s) { list.elem(s); }); + } } } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d0f298e16..22472fd72 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -6,6 +6,7 @@ #include "symbol-table.hh" #include "hash.hh" #include "config.hh" +#include "function-trace.hh" #include <map> #include <unordered_map> @@ -81,7 +82,7 @@ public: /* The allowed filesystem paths in restricted or pure evaluation mode. */ - std::experimental::optional<PathSet> allowedPaths; + std::optional<PathSet> allowedPaths; Value vEmptySet; @@ -316,6 +317,9 @@ private: /* Return a string representing the type of the value `v'. */ string showType(const Value & v); +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<string, string> decodeContext(const string & s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); @@ -346,6 +350,9 @@ struct EvalSettings : Config Setting<Strings> allowedUris{this, {}, "allowed-uris", "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/function-trace.hh b/src/libexpr/function-trace.hh new file mode 100644 index 000000000..8b0ec848d --- /dev/null +++ b/src/libexpr/function-trace.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "eval.hh" +#include <sys/time.h> + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + + FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); + } + + ~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); + } +}; +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index d38ed2df3..21a4d7917 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -295,7 +295,7 @@ static bool getDerivation(EvalState & state, Value & v, } -std::experimental::optional<DrvInfo> getDerivation(EvalState & state, Value & v, +std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 4d9128e3f..d7860fc6a 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -44,7 +44,7 @@ public: string queryDrvPath() const; string queryOutPath() const; string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `mesa.outputsToInstall`. */ + /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); @@ -78,7 +78,7 @@ typedef list<DrvInfo> DrvInfos; /* If value `v' denotes a derivation, return a DrvInfo object describing it. Otherwise return nothing. */ -std::experimental::optional<DrvInfo> getDerivation(EvalState & state, +std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); void getDerivations(EvalState & state, Value & v, const string & pathPrefix, diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 3f6017957..8bae986f9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -111,9 +111,9 @@ static void parseJSON(EvalState & state, const char * & s, Value & v) mkFloat(v, stod(tmp_number)); else mkInt(v, stol(tmp_number)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw JSONParseError("invalid JSON number"); - } catch (std::out_of_range e) { + } catch (std::out_of_range & e) { throw JSONParseError("out-of-range JSON number"); } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 29ca327c1..c34e5c383 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -6,12 +6,14 @@ %option nounput noyy_top_state +%s DEFAULT %x STRING %x IND_STRING -%x INSIDE_DOLLAR_CURLY %{ +#include <boost/lexical_cast.hpp> + #include "nixexpr.hh" #include "parser-tab.hh" @@ -97,8 +99,6 @@ URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~ %% -<INITIAL,INSIDE_DOLLAR_CURLY>{ - if { return IF; } then { return THEN; } @@ -124,9 +124,11 @@ or { return OR_KW; } {ID} { yylval->id = strdup(yytext); return ID; } {INT} { errno = 0; - yylval->n = strtol(yytext, 0, 10); - if (errno != 0) + try { + yylval->n = boost::lexical_cast<int64_t>(yytext); + } catch (const boost::bad_lexical_cast &) { throw ParseError(format("invalid integer '%1%'") % yytext); + } return INT; } {FLOAT} { errno = 0; @@ -136,17 +138,19 @@ or { return OR_KW; } return FLOAT; } -\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } -} +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } -\} { return '}'; } -<INSIDE_DOLLAR_CURLY>\} { POP_STATE(); return '}'; } -\{ { return '{'; } -<INSIDE_DOLLAR_CURLY>\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return '{'; } +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; + } +\{ { PUSH_STATE(DEFAULT); return '{'; } -<INITIAL,INSIDE_DOLLAR_CURLY>\" { - PUSH_STATE(STRING); return '"'; - } +\" { PUSH_STATE(STRING); return '"'; } <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | <STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { /* It is impossible to match strings ending with '$' with one @@ -155,7 +159,7 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext, yyleng); return STR; } -<STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <STRING>\" { POP_STATE(); return '"'; } <STRING>\$|\\|\$\\ { /* This can only occur when we reach EOF, otherwise the above @@ -165,7 +169,7 @@ or { return OR_KW; } return STR; } -<INITIAL,INSIDE_DOLLAR_CURLY>\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->e = new ExprIndStr(yytext); return IND_STR; @@ -183,14 +187,13 @@ or { return OR_KW; } yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } -<IND_STRING>\$\{ { PUSH_STATE(INSIDE_DOLLAR_CURLY); return DOLLAR_CURLY; } +<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } <IND_STRING>\' { yylval->e = new ExprIndStr("'"); return IND_STR; } -<INITIAL,INSIDE_DOLLAR_CURLY>{ {PATH} { if (yytext[yyleng-1] == '/') throw ParseError("path '%s' has a trailing slash", yytext); @@ -215,7 +218,5 @@ or { return OR_KW; } return (unsigned char) yytext[0]; } -} - %% diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index daa3258f0..ccd5293e4 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,7 +6,7 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_LIBS = libutil libstore libformat +libexpr_LIBS = libutil libstore libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/nix-expr.pc.in b/src/libexpr/nix-expr.pc.in index 79f3e2f45..80f7a492b 100644 --- a/src/libexpr/nix-expr.pc.in +++ b/src/libexpr/nix-expr.pc.in @@ -7,4 +7,4 @@ Description: Nix Package Manager Version: @PACKAGE_VERSION@ Requires: nix-store bdw-gc Libs: -L${libdir} -lnixexpr -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index cbd576d7d..967c88d9b 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,7 +1,7 @@ %glr-parser %pure-parser %locations -%error-verbose +%define parse.error verbose %defines /* %no-lines */ %parse-param { void * scanner } @@ -81,6 +81,8 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); @@ -102,11 +104,29 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, attrs = nested; } } + // Expr insertion. + // ========================== if (i->symbol.set()) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - dupAttr(attrPath, pos, j->second.pos); + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast<ExprAttrs *>(e); + auto jAttrs = dynamic_cast<ExprAttrs *>(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } } else { + // This attr path is not defined. Let's create it. attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); e->setName(i->symbol); } @@ -657,7 +677,9 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - res = { true, getDownloader()->downloadCached(store, elem.second, true) }; + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8ace6db4d..350dba474 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -315,6 +315,12 @@ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Valu mkBool(v, args[0]->type == tBool); } +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} struct CompareValues { @@ -555,7 +561,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; - std::experimental::optional<std::string> outputHash; + std::optional<std::string> outputHash; std::string outputHashAlgo; bool outputHashRecursive = false; @@ -687,21 +693,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } - /* See prim_unsafeDiscardOutputDependency. */ - else if (path.at(0) == '~') - drv.inputSrcs.insert(string(path, 1)); - /* Handle derivation outputs of the form ‘!<name>!<path>’. */ else if (path.at(0) == '!') { std::pair<string, string> ctx = decodeContext(path); drv.inputDrvs[ctx.first].insert(ctx.second); } - /* Handle derivation contexts returned by - ‘builtins.storePath’. */ - else if (isDerivation(path)) - drv.inputDrvs[path] = state.store->queryDerivationOutputNames(path); - /* Otherwise it's a source file. */ else drv.inputSrcs.insert(path); @@ -724,16 +721,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); - HashType ht = parseHashType(outputHashAlgo); - if (ht == htUnknown) - throw EvalError(format("unknown hash algorithm '%1%', at %2%") % outputHashAlgo % posDrvName); + HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); - outputHash = h.to_string(Base16, false); - if (outputHashRecursive) outputHashAlgo = "r:" + outputHashAlgo; Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); if (!jsonObject) drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, outputHashAlgo, *outputHash); + drv.outputs["out"] = DerivationOutput(outPath, + (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false)); } else { @@ -837,8 +832,14 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - if (!context.empty()) - throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + try { mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { @@ -866,7 +867,7 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - Path dir = dirOf(state.coerceToPath(pos, *args[0], context)); + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } @@ -928,6 +929,20 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); +} + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1006,13 +1021,8 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu PathSet refs; for (auto path : context) { - if (path.at(0) == '=') path = string(path, 1); - if (isDerivation(path)) { - /* See prim_unsafeDiscardOutputDependency. */ - if (path.at(0) != '~') - throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); - path = string(path, 1); - } + if (path.at(0) != '/') + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); refs.insert(path); } @@ -1680,6 +1690,8 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else @@ -1689,6 +1701,8 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else @@ -1698,6 +1712,8 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); if (args[0]->type == tFloat || args[1]->type == tFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else @@ -1707,6 +1723,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + NixFloat f2 = state.forceFloat(*args[1], pos); if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); @@ -1787,41 +1806,6 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args } -static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); -} - - -static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); -} - - -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ -static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - - PathSet context2; - for (auto & p : context) - context2.insert(p.at(0) == '=' ? "~" + string(p, 1) : p); - - mkString(v, s, context2); -} - - /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2072,9 +2056,9 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, const std::string & defaultName) { - string url; - Hash expectedHash; - string name = defaultName; + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; state.forceValue(*args[0]); @@ -2085,27 +2069,27 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, for (auto & attr : *args[0]->attrs) { string n(attr.name); if (n == "url") - url = state.forceStringNoCtx(*attr.value, *attr.pos); + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") - expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); } - if (url.empty()) + if (request.uri.empty()) throw EvalError(format("'url' argument required, at %1%") % pos); } else - url = state.forceStringNoCtx(*args[0], pos); + request.uri = state.forceStringNoCtx(*args[0], pos); - state.checkURI(url); + state.checkURI(request.uri); - if (evalSettings.pureEval && !expectedHash) + if (evalSettings.pureEval && !request.expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - Path res = getDownloader()->downloadCached(state.store, url, unpack, name, expectedHash); + Path res = getDownloader()->downloadCached(state.store, request).path; if (state.allowedPaths) state.allowedPaths->insert(res); @@ -2211,6 +2195,7 @@ void EvalState::createBaseEnv() addPrimOp("__isInt", 1, prim_isInt); addPrimOp("__isFloat", 1, prim_isFloat); addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); addPrimOp("__genericClosure", 1, prim_genericClosure); addPrimOp("abort", 1, prim_abort); addPrimOp("__addErrorContext", 2, prim_addErrorContext); @@ -2237,6 +2222,7 @@ void EvalState::createBaseEnv() addPrimOp("__readFile", 1, prim_readFile); addPrimOp("__readDir", 1, prim_readDir); addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); @@ -2292,9 +2278,6 @@ void EvalState::createBaseEnv() addPrimOp("toString", 1, prim_toString); addPrimOp("__substring", 3, prim_substring); addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hasContext", 1, prim_hasContext); - addPrimOp("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); - addPrimOp("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); addPrimOp("__hashString", 2, prim_hashString); addPrimOp("__match", 2, prim_match); addPrimOp("__split", 2, prim_split); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc new file mode 100644 index 000000000..2d79739ea --- /dev/null +++ b/src/libexpr/primops/context.cc @@ -0,0 +1,187 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "derivations.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + + +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto & p : context) + context2.insert(p.at(0) == '=' ? string(p, 1) : p); + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present. +*/ +static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map<Path, ContextInfo>(); + for (const auto & p : context) { + Path drv; + string output; + const Path * path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair<string, string> ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto & info : contextInfos) { + auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) + mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto & output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } + } + infoVal.attrs->sort(); + } + v.attrs->sort(); +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto & i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + if (!settings.readOnlyMode) + state.store->ensurePath(i.name); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) + context.insert(i.name); + } + + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + context.insert("=" + string(i.name)); + } + } + + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + +} diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 7aa98e0bf..90f600284 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -3,6 +3,7 @@ #include "download.hh" #include "store-api.hh" #include "pathlocks.hh" +#include "hash.hh" #include <sys/time.h> @@ -25,7 +26,7 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref<Store> store, const std::string & uri, - std::experimental::optional<std::string> ref, std::string rev, + std::optional<std::string> ref, std::string rev, const std::string & name) { if (evalSettings.pureEval && rev == "") @@ -37,7 +38,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, try { runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError e) { + } catch (ExecError & e) { if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; clean = false; } @@ -84,15 +85,20 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, if (rev != "" && !std::regex_match(rev, revRegex)) throw Error("invalid Git revision '%s'", rev); - Path cacheDir = getCacheDir() + "/nix/git"; + deletePath(getCacheDir() + "/nix/git"); + + Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); runProgram("git", true, { "init", "--bare", cacheDir }); } - std::string localRef = hashString(htSHA256, fmt("%s-%s", uri, *ref)).to_string(Base32, false); - - Path localRefFile = cacheDir + "/refs/heads/" + localRef; + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; bool doFetch; time_t now = time(0); @@ -114,7 +120,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now; + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; } if (doFetch) { @@ -122,7 +128,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, *ref + ":" + localRef }); + runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); struct timeval times[2]; times[0].tv_sec = now; @@ -188,7 +194,7 @@ GitInfo exportGit(ref<Store> store, const std::string & uri, static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::experimental::optional<std::string> ref; + std::optional<std::string> ref; std::string rev; std::string name = "source"; PathSet context; @@ -219,8 +225,6 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va } else url = state.coerceToString(pos, *args[0], context, false, false); - if (!isUri(url)) url = absPath(url); - // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. state.checkURI(url); @@ -235,7 +239,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(gitInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 9d35f6d0d..a907d0e1c 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -80,7 +80,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, time_t now = time(0); struct stat st; if (stat(stampFile.c_str(), &st) != 0 || - st.st_mtime + settings.tarballTtl <= now) + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) { /* Except that if this is a commit hash that we already have, we don't have to pull again. */ @@ -93,7 +93,19 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); if (pathExists(cacheDir)) { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } } else { createDirs(dirOf(cacheDir)); runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); @@ -184,8 +196,6 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar } else url = state.coerceToString(pos, *args[0], context, false, false); - if (!isUri(url)) url = absPath(url); - // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. state.checkURI(url); @@ -201,7 +211,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(hgInfo.storePath); + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4128de05d..a84e569e9 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -49,6 +49,19 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); } + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + else if (t->is_value()) { if (auto val = t->as<int64_t>()) mkInt(v, val->get()); diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 44929f7ee..91faea122 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -75,6 +75,13 @@ public: } size_t totalSize() const; + + template<typename T> + void dump(T callback) + { + for (auto & s : symbols) + callback(s); + } }; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 809772f7c..e1ec87d3b 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -43,7 +43,7 @@ class XMLWriter; class JSONPlaceholder; -typedef long NixInt; +typedef int64_t NixInt; typedef double NixFloat; /* External values must descend from ExternalValueBase, so that diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 4c35a4199..9e1d7cee6 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -35,6 +35,15 @@ MixCommonArgs::MixCommonArgs(const string & programName) } }); + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { + settings.set("max-jobs", s); + }); + std::string cat = "config"; globalConfig.convertToArgs(*this, cat); diff --git a/src/libmain/local.mk b/src/libmain/local.mk index f1fd3eb72..0c80f5a0a 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -8,7 +8,7 @@ libmain_SOURCES := $(wildcard $(d)/*.cc) libmain_LDFLAGS = $(OPENSSL_LIBS) -libmain_LIBS = libstore libutil libformat +libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 diff --git a/src/libmain/nix-main.pc.in b/src/libmain/nix-main.pc.in index 38bc85c48..37b03dcd4 100644 --- a/src/libmain/nix-main.pc.in +++ b/src/libmain/nix-main.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixmain -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4ed34e54d..d3dbfbc44 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -80,6 +80,7 @@ string getArg(const string & opt, } +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* OpenSSL is not thread-safe by default - it will randomly crash unless the user supplies a mutex locking function. So let's do that. */ @@ -92,6 +93,7 @@ static void opensslLockCallback(int mode, int type, const char * file, int line) else opensslLocks[type].unlock(); } +#endif static void sigHandler(int signo) { } @@ -105,9 +107,11 @@ void initNix() std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); #endif +#if OPENSSL_VERSION_NUMBER < 0x10101000L /* Initialise OpenSSL locking. */ opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); CRYPTO_set_locking_callback(opensslLockCallback); +#endif loadConfFile(); @@ -125,6 +129,15 @@ void initNix() act.sa_handler = sigHandler; if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif + /* Register a SIGSEGV handler to detect stack overflows. */ detectStackOverflow(); @@ -175,10 +188,6 @@ LegacyArgs::LegacyArgs(const std::string & programName, .description("build from source if substitution fails") .set(&(bool&) settings.tryFallback, true); - mkFlag1('j', "max-jobs", "jobs", "maximum number of parallel builds", [=](std::string s) { - settings.set("max-jobs", s); - }); - auto intSettingAlias = [&](char shortName, const std::string & longName, const std::string & description, const std::string & dest) { mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index 13896aeec..e6224de7d 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -63,7 +63,7 @@ void detectStackOverflow() act.sa_sigaction = sigsegvHandler; act.sa_flags = SA_SIGINFO | SA_ONSTACK; if (sigaction(SIGSEGV, &act, 0)) - throw SysError("resetting SIGCHLD"); + throw SysError("resetting SIGSEGV"); #endif } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9c75c8599..8e6f1f55d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -10,10 +10,13 @@ #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" #include "json.hh" +#include "thread-pool.hh" #include <chrono> - #include <future> +#include <regex> + +#include <nlohmann/json.hpp> namespace nix { @@ -55,7 +58,7 @@ void BinaryCacheStore::init() } void BinaryCacheStore::getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) + Callback<std::shared_ptr<std::string>> callback) noexcept { try { callback(getFile(path)); @@ -139,6 +142,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(info.path, *nar, narAccessor); + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -148,11 +156,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str JSONObject jsonRoot(jsonOut); jsonRoot.attr("version", 1); - auto narAccessor = makeNarAccessor(nar); - - if (accessor_) - accessor_->addToCache(info.path, *nar, narAccessor); - { auto res = jsonRoot.placeholder("root"); listNar(res, narAccessor, "", true); @@ -162,11 +165,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); } - else { - if (accessor_) - accessor_->addToCache(info.path, *nar, makeNarAccessor(nar)); - } - /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); @@ -181,12 +179,70 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) % duration); - /* Atomically write the NAR file. */ narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : ""); + + /* Optionally maintain an index of DWARF debug info files + consisting of JSON files named 'debuginfo/<build-id>' that + specify the NAR file and member containing the debug info. */ + if (writeDebugInfo) { + + std::string buildIdDir = "/lib/debug/.build-id"; + + if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + + ThreadPool threadPool(25); + + auto doFile = [&](std::string member, std::string key, std::string target) { + checkInterrupt(); + + nlohmann::json json; + json["archive"] = target; + json["member"] = member; + + // FIXME: or should we overwrite? The previous link may point + // to a GC'ed file, so overwriting might be useful... + if (fileExists(key)) return; + + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + + upsertFile(key, json.dump(), "application/json"); + }; + + std::regex regex1("^[0-9a-f]{2}$"); + std::regex regex2("^[0-9a-f]{38}\\.debug$"); + + for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + "/" + s1; + + if (narAccessor->stat(dir).type != FSAccessor::tDirectory + || !std::regex_match(s1, regex1)) + continue; + + for (auto & s2 : narAccessor->readDirectory(dir)) { + auto debugPath = dir + "/" + s2; + + if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + || !std::regex_match(s2, regex2)) + continue; + + auto buildId = s1 + s2; + + std::string key = "debuginfo/" + buildId; + std::string target = "../" + narInfo->url; + + threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + } + } + + threadPool.process(); + } + } + + /* Atomically write the NAR file. */ if (repair || !fileExists(narInfo->url)) { stats.narWrite++; upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); @@ -232,7 +288,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) throw SubstituteGone(e.what()); } - decompressor->flush(); + decompressor->finish(); stats.narRead++; //stats.narReadCompressedBytes += nar->size(); // FIXME @@ -240,7 +296,7 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) } void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, - Callback<std::shared_ptr<ValidPathInfo>> callback) + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { auto uri = getUri(); auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, @@ -249,21 +305,23 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, auto narInfoFile = narInfoFileFor(storePath); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + getFile(narInfoFile, {[=](std::future<std::shared_ptr<std::string>> fut) { try { auto data = fut.get(); - if (!data) return callback(nullptr); + if (!data) return (*callbackPtr)(nullptr); stats.narInfoRead++; - callback((std::shared_ptr<ValidPathInfo>) + (*callbackPtr)((std::shared_ptr<ValidPathInfo>) std::make_shared<NarInfo>(*this, *data, narInfoFile)); (void) act; // force Activity into this lambda to ensure it stays alive } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 6bc83fc50..c77292294 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,6 +17,7 @@ public: const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<bool> parallelCompression{this, false, "parallel-compression", @@ -47,7 +48,7 @@ public: /* Fetch the specified file and call the specified callback with the result. A subclass may implement this asynchronously. */ virtual void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback); + Callback<std::shared_ptr<std::string>> callback) noexcept; std::shared_ptr<std::string> getFile(const std::string & path); @@ -72,24 +73,11 @@ public: bool isValidPathUncached(const Path & path) override; - PathSet queryAllValidPaths() override - { unsupported(); } - void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override; - - void queryReferrers(const Path & path, - PathSet & referrers) override - { unsupported(); } - - PathSet queryDerivationOutputs(const Path & path) override - { unsupported(); } - - StringSet queryDerivationOutputNames(const Path & path) override - { unsupported(); } + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; Path queryPathFromHashPart(const string & hashPart) override - { unsupported(); } + { unsupported("queryPathFromHashPart"); } bool wantMassQuery() override { return wantMassQuery_; } @@ -108,22 +96,10 @@ public: BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override - { unsupported(); } + { unsupported("buildDerivation"); } void ensurePath(const Path & path) override - { unsupported(); } - - void addTempRoot(const Path & path) override - { unsupported(); } - - void addIndirectRoot(const Path & path) override - { unsupported(); } - - Roots findRoots() override - { unsupported(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { unsupported(); } + { unsupported("ensurePath"); } ref<FSAccessor> getFSAccessor() override; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d75ca0be8..9a6729f9e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -11,6 +11,8 @@ #include "compression.hh" #include "json.hh" #include "nar-info.hh" +#include "parsed-derivations.hh" +#include "machines.hh" #include <algorithm> #include <iostream> @@ -20,6 +22,7 @@ #include <future> #include <chrono> #include <regex> +#include <queue> #include <limits.h> #include <sys/time.h> @@ -35,6 +38,7 @@ #include <unistd.h> #include <errno.h> #include <cstring> +#include <termios.h> #include <pwd.h> #include <grp.h> @@ -262,6 +266,12 @@ public: /* Set if at least one derivation had a timeout. */ bool timedOut; + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + LocalStore & store; std::unique_ptr<HookInstance> hook; @@ -458,6 +468,28 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } +} ////////////////////////////////////////////////////////////////////// @@ -740,6 +772,8 @@ private: /* The derivation stored at drvPath. */ std::unique_ptr<BasicDerivation> drv; + std::unique_ptr<ParsedDerivation> parsedDrv; + /* The remainder is state held during the build. */ /* Locks on the output paths. */ @@ -854,7 +888,7 @@ private: building multiple times. Since this contains the hash, it allows us to compare whether two rounds produced the same result. */ - ValidPathInfos prevInfos; + std::map<Path, ValidPathInfo> prevInfos; const uid_t sandboxUid = 1000; const gid_t sandboxGid = 100; @@ -877,6 +911,9 @@ public: Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + void timedOut() override; string key() override @@ -935,6 +972,11 @@ private: as valid. */ void registerOutputs(); + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); + /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -1024,6 +1066,17 @@ DerivationGoal::~DerivationGoal() } +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + void DerivationGoal::killChild() { if (pid != -1) { @@ -1139,10 +1192,12 @@ void DerivationGoal::haveDerivation() return; } + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - if (settings.useSubstitutes && drv->substitutesAllowed()) + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & i : invalidOutputs) addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); @@ -1395,7 +1450,7 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || drv->willBuildLocally(); + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); auto started = [&]() { auto msg = fmt( @@ -1510,8 +1565,8 @@ void DerivationGoal::buildDone() if (hook) { hook->builderOut.readSide = -1; hook->fromHook.readSide = -1; - } - else builderOut.readSide = -1; + } else + builderOut.readSide = -1; /* Close the log file. */ closeLogFile(); @@ -1574,6 +1629,61 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map<std::string, std::string> hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + if (buildMode == bmCheck) { done(BuildResult::Built); return; @@ -1641,19 +1751,13 @@ HookReply DerivationGoal::tryBuildHook() try { - /* Tell the hook about system features (beyond the system type) - required from the build machine. (The hook could parse the - drv file itself, but this is easier.) */ - Strings features = tokenizeString<Strings>(get(drv->env, "requiredSystemFeatures")); - for (auto & i : features) checkStoreName(i); /* !!! abuse */ - /* Send the request to the hook. */ worker.hook->sink << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform << drvPath - << features; + << parsedDrv->getRequiredSystemFeatures(); worker.hook->sink.flush(); /* Read the first line of input, which should be a word indicating @@ -1793,23 +1897,26 @@ static void preloadNSS() { void DerivationGoal::startBuilder() { /* Right platform? */ - if (!drv->canBuildLocally()) { - throw Error( - format("a '%1%' is required to build '%3%', but I am a '%2%'") - % drv->platform % settings.thisSystem % drvPath); - } + if (!parsedDrv->canBuildLocally()) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + drvPath, + settings.thisSystem, + concatStringsSep(", ", settings.systemFeatures)); if (drv->isBuiltin()) preloadNSS(); #if __APPLE__ - additionalSandboxProfile = get(drv->env, "__sandboxProfile"); + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif /* Are we doing a chroot build? */ { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); if (settings.sandboxMode == smEnabled) { - if (get(drv->env, "__noChroot") == "1") + if (noChroot) throw Error(format("derivation '%1%' has '__noChroot' set, " "but that's not allowed when 'sandbox' is 'true'") % drvPath); #if __APPLE__ @@ -1822,7 +1929,7 @@ void DerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !fixedOutput && get(drv->env, "__noChroot") != "1"; + useChroot = !fixedOutput && !noChroot; } if (worker.store.storeDir != worker.store.realStoreDir) { @@ -1873,7 +1980,7 @@ void DerivationGoal::startBuilder() writeStructuredAttrs(); /* Handle exportReferencesGraph(), if set. */ - if (!drv->env.count("__json")) { + if (!parsedDrv->getStructuredAttrs()) { /* The `exportReferencesGraph' feature allows the references graph to be passed to a builder. This attribute should be a list of pairs [name1 path1 name2 path2 ...]. The references graph of @@ -1938,7 +2045,7 @@ void DerivationGoal::startBuilder() PathSet allowedPaths = settings.allowedImpureHostPrefixes; /* This works like the above, except on a per-derivation level */ - Strings impurePaths = tokenizeString<Strings>(get(drv->env, "__impureHostDeps")); + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); for (auto & i : impurePaths) { bool found = false; @@ -2007,7 +2114,7 @@ void DerivationGoal::startBuilder() /* Create /etc/hosts with localhost entry. */ if (!fixedOutput) - writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n"); + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, rather than the whole Nix store. This prevents any access @@ -2064,7 +2171,7 @@ void DerivationGoal::startBuilder() #endif } - else { + if (needsHashRewrite()) { if (pathExists(homeDir)) throw Error(format("directory '%1%' exists; please remove it") % homeDir); @@ -2136,7 +2243,48 @@ void DerivationGoal::startBuilder() Path logFile = openLogFile(); /* Create a pipe to get the output of the builder. */ - builderOut.create(); + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); result.startTime = time(0); @@ -2209,17 +2357,37 @@ void DerivationGoal::startBuilder() flags |= CLONE_NEWNET; pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) + if (child == -1 && errno == EINVAL) { /* Fallback for Linux < 2.13 where CLONE_NEWPID and CLONE_PARENT are not allowed together. */ - child = clone(childEntry, stack + stackSize, flags & ~CLONE_NEWPID, this); + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unprivileged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); if (child == -1) throw SysError("cloning builder process"); writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); _exit(0); }, options); - if (helper.wait() != 0) + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + tmpDirInSandbox = tmpDir; + goto fallback; + } else if (res != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -2242,14 +2410,14 @@ void DerivationGoal::startBuilder() writeFile("/proc/" + std::to_string(pid) + "/gid_map", (format("%d %d 1") % sandboxGid % hostGid).str()); - /* Signal the builder that we've updated its user - namespace. */ + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; } else #endif { + fallback: options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); @@ -2306,7 +2474,7 @@ void DerivationGoal::initEnv() passAsFile is ignored in structure mode because it's not needed (attributes are not passed through the environment, so there is no size constraint). */ - if (!drv->env.count("__json")) { + if (!parsedDrv->getStructuredAttrs()) { StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); int fileNr = 0; @@ -2353,14 +2521,17 @@ void DerivationGoal::initEnv() fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ if (fixedOutput) { - Strings varNames = tokenizeString<Strings>(get(drv->env, "impureEnvVars")); - for (auto & i : varNames) env[i] = getEnv(i); + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i); } /* Currently structured log messages piggyback on stderr, but we may change that in the future. So tell the builder which file descriptor to use for that. */ env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; } @@ -2369,111 +2540,103 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); void DerivationGoal::writeStructuredAttrs() { - auto jsonAttr = drv->env.find("__json"); - if (jsonAttr == drv->env.end()) return; + auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; - try { + auto json = *structuredAttrs; - auto jsonStr = rewriteStrings(jsonAttr->second, inputRewrites); - - auto json = nlohmann::json::parse(jsonStr); - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - PathSet storePaths; - for (auto & p : *i) - storePaths.insert(p.get<std::string>()); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get<std::string>()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh } + } - writeFile(tmpDir + "/.attrs.json", json.dump()); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - auto handleSimpleType = [](const nlohmann::json & value) -> std::experimental::optional<std::string> { - if (value.is_string()) - return shellEscape(value); + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ - if (value.is_number()) { - auto f = value.get<float>(); - if (std::ceil(f) == f) - return std::to_string(value.get<int>()); - } + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { + if (value.is_string()) + return shellEscape(value); - if (value.is_null()) - return std::string("''"); + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); + } - if (value.is_boolean()) - return value.get<bool>() ? std::string("1") : std::string(""); + if (value.is_null()) + return std::string("''"); - return {}; - }; + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); - std::string jsonSh; + return {}; + }; - for (auto i = json.begin(); i != json.end(); ++i) { + std::string jsonSh; - if (!std::regex_match(i.key(), shVarName)) continue; + for (auto i = json.begin(); i != json.end(); ++i) { - auto & value = i.value(); + if (!std::regex_match(i.key(), shVarName)) continue; - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); + auto & value = i.value(); - else if (value.is_array()) { - std::string s2; - bool good = true; + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } + else if (value.is_array()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; } - else if (value.is_object()) { - std::string s2; - bool good = true; + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } + else if (value.is_object()) { + std::string s2; + bool good = true; - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); } - } - - writeFile(tmpDir + "/.attrs.sh", jsonSh); - } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); } @@ -2499,17 +2662,17 @@ void setupSeccomp() seccomp_release(ctx); }); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); - if (settings.thisSystem == "x86_64-linux" && + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) throw SysError("unable to add X32 seccomp architecture"); - if (settings.thisSystem == "aarch64-linux" && + if (nativeSystem == "aarch64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) - printError("unsable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes."); + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { @@ -2628,7 +2791,7 @@ void DerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (pathExists("/dev/kvm")) + if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); @@ -2646,7 +2809,13 @@ void DerivationGoal::runChild() on. */ if (fixedOutput) { ss.push_back("/etc/resolv.conf"); - ss.push_back("/etc/nsswitch.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + ss.push_back("/etc/services"); ss.push_back("/etc/hosts"); if (pathExists("/var/run/nscd/socket")) @@ -2872,6 +3041,10 @@ void DerivationGoal::runChild() for (auto & i : missingPaths) { sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); } + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); + } sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -2917,7 +3090,7 @@ void DerivationGoal::runChild() writeFile(sandboxFile, sandboxProfile); - bool allowLocalNetworking = get(drv->env, "__darwinAllowLocalNetworking") == "1"; + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ @@ -2989,10 +3162,9 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, string attr) +PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) { PathSet result; - Paths paths = tokenizeString<Paths>(attr); for (auto & i : paths) { if (store.isStorePath(i)) result.insert(i); @@ -3017,7 +3189,7 @@ void DerivationGoal::registerOutputs() if (allValid) return; } - ValidPathInfos infos; + std::map<std::string, ValidPathInfo> infos; /* Set of inodes seen during calls to canonicalisePathMetaData() for this build's outputs. This needs to be shared between @@ -3025,8 +3197,7 @@ void DerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool runDiffHook = settings.runDiffHook; - bool keepPreviousRound = settings.keepFailed || runDiffHook; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -3051,7 +3222,9 @@ void DerivationGoal::registerOutputs() throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); - } else { + } + + if (needsHashRewrite()) { Path redirected = redirectedOutputs[path]; if (buildMode == bmRepair && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() @@ -3121,15 +3294,16 @@ void DerivationGoal::registerOutputs() the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - Path dest = worker.store.makeFixedOutputPath(recursive, h2, drv->env["name"]); + Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); if (h != h2) { /* Throw an error after registering the path as valid. */ + worker.hashMismatch = true; delayedException = std::make_exception_ptr( - BuildError("fixed-output derivation produced path '%s' with %s hash '%s' instead of the expected hash '%s'", - dest, printHashType(h.type), printHash16or32(h2), printHash16or32(h))); + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + dest, h.to_string(), h2.to_string())); Path actualDest = worker.store.toRealPath(dest); @@ -3169,15 +3343,22 @@ void DerivationGoal::registerOutputs() if (!worker.store.isValidPath(path)) continue; auto info = *worker.store.queryPathInfo(path); if (hash.first != info.narHash) { - if (settings.keepFailed) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); if (rename(actualPath.c_str(), dst.c_str())) throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + path, dst, drvPath, tmpDir); + + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") % drvPath % path % dst); } else - throw Error(format("derivation '%1%' may not be deterministic: output '%2%' differs") + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") % drvPath % path); } @@ -3202,48 +3383,6 @@ void DerivationGoal::registerOutputs() debug(format("referenced input: '%1%'") % i); } - /* Enforce `allowedReferences' and friends. */ - auto checkRefs = [&](const string & attrName, bool allowed, bool recursive) { - if (drv->env.find(attrName) == drv->env.end()) return; - - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, get(drv->env, attrName)); - - PathSet used; - if (recursive) { - /* Our requisites are the union of the closures of our references. */ - for (auto & i : references) - /* Don't call computeFSClosure on ourselves. */ - if (path != i) - worker.store.computeFSClosure(i, used); - } else - used = references; - - PathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (spec.find(i) == spec.end()) - badPaths.insert(i); - } else { - if (spec.find(i) != spec.end()) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n\t"; - badPathsStr += i; - } - throw BuildError(format("output '%1%' is not allowed to refer to the following paths:%2%") % actualPath % badPathsStr); - } - }; - - checkRefs("allowedReferences", true, false); - checkRefs("allowedRequisites", true, true); - checkRefs("disallowedReferences", false, false); - checkRefs("disallowedRequisites", false, true); - if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(path); @@ -3259,11 +3398,14 @@ void DerivationGoal::registerOutputs() if (!info.references.empty()) info.ca.clear(); - infos.push_back(info); + infos[i.first] = info; } if (buildMode == bmCheck) return; + /* Apply output checks. */ + checkOutputs(infos); + /* Compare the result with the previous round, and report which path is different, if any.*/ if (curRound > 1 && prevInfos != infos) { @@ -3271,22 +3413,16 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->path + checkSuffix; + Path prev = i->second.path + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->path, drvPath); - - auto diffHook = settings.diffHook; - if (prevExists && diffHook != "" && runDiffHook) { - try { - auto diff = runProgram(diffHook, true, {prev, i->path}); - if (diff != "") - printError(chomp(diff)); - } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); - } - } + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, i->second.path, drvPath, tmpDir); if (settings.enforceDeterminism) throw NotDeterministic(msg); @@ -3325,7 +3461,11 @@ void DerivationGoal::registerOutputs() /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ - worker.store.registerValidPaths(infos); + { + ValidPathInfos infos2; + for (auto & i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } /* In case of a fixed-output derivation hash mismatch, throw an exception now that we have registered the output as valid. */ @@ -3334,6 +3474,158 @@ void DerivationGoal::registerOutputs() } +void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) +{ + std::map<Path, const ValidPathInfo &> outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(output.second.path, output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::optional<uint64_t> maxSize, maxClosureSize; + std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path & path) + { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue<Path> pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(pathsDone, closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + info.path, info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + info.path, closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive) + { + if (!value) return; + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + PathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get<uint64_t>(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get<uint64_t>(); + + auto get = [&](const std::string & name) -> std::optional<Strings> { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + Path DerivationGoal::openLogFile() { logSize = 0; @@ -3682,6 +3974,19 @@ void SubstitutionGoal::tryNext() } catch (InvalidPath &) { tryNext(); return; + } catch (SubstituterDisabled &) { + if (settings.tryFallback) { + tryNext(); + return; + } + throw; + } catch (Error & e) { + if (settings.tryFallback) { + printError(e.what()); + tryNext(); + return; + } + throw; } /* Update the total expected download size. */ @@ -3754,17 +4059,6 @@ void SubstitutionGoal::tryToRun() return; } - /* If the store path is already locked (probably by a - DerivationGoal), then put this goal to sleep. Note: we don't - acquire a lock here since that breaks addToStore(), so below we - handle an AlreadyLocked exception from addToStore(). The check - here is just an optimisation to prevent having to redo a - download due to a locked path. */ - if (pathIsLockedByMe(worker.store.toRealPath(storePath))) { - worker.waitForAWhile(shared_from_this()); - return; - } - maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); worker.updateProgress(); @@ -3804,12 +4098,6 @@ void SubstitutionGoal::finished() try { promise.get_future().get(); - } catch (AlreadyLocked & e) { - /* Probably a DerivationGoal is already building this store - path. Sleep for a while and try again. */ - state = &SubstitutionGoal::init; - worker.waitForAWhile(shared_from_this()); - return; } catch (std::exception & e) { printError(e.what()); @@ -3885,6 +4173,8 @@ Worker::Worker(LocalStore & store) lastWokenUp = steady_time_point::min(); permanentFailure = false; timedOut = false; + hashMismatch = false; + checkMismatch = false; } @@ -4190,14 +4480,15 @@ void Worker::waitForInput() for (auto & k : fds2) { if (FD_ISSET(k, &fds)) { ssize_t rd = read(k, buffer.data(), buffer.size()); - if (rd == -1) { - if (errno != EINTR) - throw SysError(format("reading from %1%") - % goal->getName()); - } else if (rd == 0) { + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { debug(format("%1%: got EOF") % goal->getName()); goal->handleEOF(k); j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); } else { printMsg(lvlVomit, format("%1%: read %2% bytes") % goal->getName() % rd); @@ -4244,7 +4535,29 @@ void Worker::waitForInput() unsigned int Worker::exitStatus() { - return timedOut ? 101 : (permanentFailure ? 100 : 1); + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 + if (timedOut) + mask |= 0x01; // 101 + if (hashMismatch) + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) + mask |= 0x60; + return mask ? mask : 1; } @@ -4282,6 +4595,11 @@ static void primeCache(Store & store, const PathSet & paths) PathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index b4dcb35f9..b1af3b4fc 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -24,6 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) Path storePath = getAttr("out"); auto mainUrl = getAttr("url"); + bool unpack = get(drv.env, "unpack", "") == "1"; /* Note: have to use a fresh downloader here because we're in a forked process. */ @@ -40,12 +41,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) request.decompress = false; auto decompressor = makeDecompressionSink( - hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); downloader->download(std::move(request), *decompressor); decompressor->finish(); }); - if (get(drv.env, "unpack", "") == "1") + if (unpack) restorePath(storePath, *source); else writeFile(storePath, *source); @@ -63,7 +64,8 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); - fetch(hashedMirror + printHashType(ht) + "/" + Hash(getAttr("outputHash"), ht).to_string(Base16, false)); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1e187ec5e..23fcfb281 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -36,32 +36,12 @@ Path BasicDerivation::findOutput(const string & id) const } -bool BasicDerivation::willBuildLocally() const -{ - return get(env, "preferLocalBuild") == "1" && canBuildLocally(); -} - - -bool BasicDerivation::substitutesAllowed() const -{ - return get(env, "allowSubstitutes", "1") == "1"; -} - - bool BasicDerivation::isBuiltin() const { return string(builder, 0, 8) == "builtin:"; } -bool BasicDerivation::canBuildLocally() const -{ - return platform == settings.thisSystem - || settings.extraPlatforms.get().count(platform) > 0 - || isBuiltin(); -} - - Path writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 7b97730d3..8e02c9bc5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -56,14 +56,8 @@ struct BasicDerivation the given derivation. */ Path findOutput(const string & id) const; - bool willBuildLocally() const; - - bool substitutesAllowed() const; - bool isBuiltin() const; - bool canBuildLocally() const; - /* Return true iff this is a fixed-output derivation. */ bool isFixedOutput() const; diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 973fca0b1..cdf56e09d 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -30,23 +30,7 @@ using namespace std::string_literals; namespace nix { -struct DownloadSettings : Config -{ - Setting<bool> enableHttp2{this, true, "http2", - "Whether to enable HTTP/2 support."}; - - Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", - "String appended to the user agent in HTTP requests."}; - - Setting<size_t> httpConnections{this, 25, "http-connections", - "Number of parallel HTTP connections.", - {"binary-caches-parallel-connections"}}; - - Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", - "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; -}; - -static DownloadSettings downloadSettings; +DownloadSettings downloadSettings; static GlobalConfig::Register r1(&downloadSettings); @@ -87,19 +71,24 @@ struct CurlDownloader : public Downloader std::string encoding; + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + DownloadItem(CurlDownloader & downloader, const DownloadRequest & request, - Callback<DownloadResult> callback) + Callback<DownloadResult> && callback) : downloader(downloader) , request(request) , act(*logger, lvlTalkative, actDownload, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) - , callback(callback) + , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { - if (this->request.dataCallback) + if (this->request.dataCallback) { + writtenToSink += len; this->request.dataCallback((char *) data, len); - else + } else this->result.data->append((char *) data, len); }) { @@ -177,6 +166,7 @@ struct CurlDownloader : public Downloader status = ss.size() >= 2 ? ss[1] : ""; result.data = std::make_shared<std::string>(); result.bodySize = 0; + acceptRanges = false; encoding = ""; } else { auto i = line.find(':'); @@ -194,7 +184,9 @@ struct CurlDownloader : public Downloader return 0; } } else if (name == "content-encoding") - encoding = trim(string(line, i + 1));; + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; } } return realSize; @@ -244,8 +236,6 @@ struct CurlDownloader : public Downloader return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); } - long lowSpeedTimeout = 300; - void init() { if (!req) req = curl_easy_init(); @@ -270,6 +260,8 @@ struct CurlDownloader : public Downloader #if LIBCURL_VERSION_NUM >= 0x072f00 if (downloadSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); @@ -303,13 +295,16 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, lowSpeedTimeout); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + result.data = std::make_shared<std::string>(); result.bodySize = 0; } @@ -319,16 +314,21 @@ struct CurlDownloader : public Downloader long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); - char * effectiveUrlCStr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUrlCStr); - if (effectiveUrlCStr) - result.effectiveUrl = effectiveUrlCStr; + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) - decompressionSink->finish(); + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { code = CURLE_OK; @@ -339,18 +339,12 @@ struct CurlDownloader : public Downloader failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); done = true; - - try { - act.progress(result.data->size(), result.data->size()); - callback(std::move(result)); - } catch (...) { - done = true; - callback.rethrow(); - } + callback(std::move(result)); } else { @@ -412,10 +406,20 @@ struct CurlDownloader : public Downloader request.verb(), request.uri, curl_easy_strerror(code), code)); /* If this is a transient error, then maybe retry the - download after a while. */ - if (err == Transient && attempt < request.tries) { + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient + && attempt < request.tries + && (!this->request.dataCallback + || writtenToSink == 0 + || (acceptRanges && encoding.empty()))) + { int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); - printError(format("warning: %s; retrying in %d ms") % exc.what() % ms); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); downloader.enqueueItem(shared_from_this()); } @@ -528,10 +532,11 @@ struct CurlDownloader : public Downloader extraFDs[0].fd = wakeupPipe.readSide.get(); extraFDs[0].events = CURL_WAIT_POLLIN; extraFDs[0].revents = 0; + long maxSleepTimeMs = items.empty() ? 10000 : 100; auto sleepTimeMs = nextWakeup != std::chrono::steady_clock::time_point() ? std::max(0, (int) std::chrono::duration_cast<std::chrono::milliseconds>(nextWakeup - std::chrono::steady_clock::now()).count()) - : 10000; + : maxSleepTimeMs; vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) @@ -587,7 +592,7 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError(format("unexpected error in download thread: %s") % e.what()); + printError("unexpected error in download thread: %s", e.what()); } { @@ -613,6 +618,22 @@ struct CurlDownloader : public Downloader writeFull(wakeupPipe.writeSide.get(), " "); } +#ifdef ENABLE_S3 + std::tuple<std::string, std::string, Store::Params> parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + void enqueueDownload(const DownloadRequest & request, Callback<DownloadResult> callback) override { @@ -621,12 +642,15 @@ struct CurlDownloader : public Downloader // FIXME: do this on a worker thread try { #ifdef ENABLE_S3 - S3Helper s3Helper("", Aws::Region::US_EAST_1, ""); // FIXME: make configurable - auto slash = request.uri.find('/', 5); - if (slash == std::string::npos) - throw nix::Error("bad S3 URI '%s'", request.uri); - std::string bucketName(request.uri, 5, slash - 5); - std::string key(request.uri, slash + 1); + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); DownloadResult res; @@ -641,7 +665,7 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared<DownloadItem>(*this, request, callback)); + enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); } }; @@ -710,11 +734,12 @@ void Downloader::download(DownloadRequest && request, Sink & sink) /* If the buffer is full, then go to sleep until the calling thread wakes us up (i.e. when it has removed data from the - buffer). Note: this does stall the download thread. */ - while (state->data.size() > 1024 * 1024) { - if (state->quit) return; + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { debug("download buffer is full; going to sleep"); - state.wait(state->request); + state.wait_for(state->request, std::chrono::seconds(10)); } /* Append data to the buffer and wake up the calling @@ -736,47 +761,59 @@ void Downloader::download(DownloadRequest && request, Sink & sink) state->request.notify_one(); }}); - auto state(_state->lock()); - while (true) { checkInterrupt(); - /* If no data is available, then wait for the download thread - to wake us up. */ - if (state->data.empty()) { + std::string chunk; - if (state->quit) { - if (state->exc) std::rethrow_exception(state->exc); - break; + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + while (state->data.empty()) { + + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + return; + } + + state.wait(state->avail); } - state.wait(state->avail); - } + chunk = std::move(state->data); - /* If data is available, then flush it to the sink and wake up - the download thread if it's blocked on a full buffer. */ - if (!state->data.empty()) { - sink((unsigned char *) state->data.data(), state->data.size()); - state->data.clear(); state->request.notify_one(); } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink((unsigned char *) chunk.data(), chunk.size()); } } -Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash, string * effectiveUrl, int ttl) +CachedDownloadResult Downloader::downloadCached( + ref<Store> store, const CachedDownloadRequest & request) { - auto url = resolveUri(url_); + auto url = resolveUri(request.uri); + auto name = request.name; if (name == "") { auto p = url.rfind('/'); if (p != string::npos) name = string(url, p + 1); } Path expectedStorePath; - if (expectedHash) { - expectedStorePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(expectedStorePath)) - return store->toRealPath(expectedStorePath); + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + return result; + } } Path cacheDir = getCacheDir() + "/nix/tarballs"; @@ -795,6 +832,8 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa bool skip = false; + CachedDownloadResult result; + if (pathExists(fileLink) && pathExists(dataFile)) { storePath = readLink(fileLink); store->addTempRoot(storePath); @@ -802,10 +841,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; - if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0)) { + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { skip = true; - if (effectiveUrl) - *effectiveUrl = url_; + result.effectiveUri = request.uri; + result.etag = ss[1]; } else if (!ss[1].empty()) { debug(format("verifying previous ETag '%1%'") % ss[1]); expectedETag = ss[1]; @@ -818,17 +857,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa if (!skip) { try { - DownloadRequest request(url); - request.expectedETag = expectedETag; - auto res = download(request); - if (effectiveUrl) - *effectiveUrl = res.effectiveUrl; + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; if (!res.cached) { ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); - Hash hash = hashString(expectedHash ? expectedHash.type : htSHA256, *res.data); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); info.path = store->makeFixedOutputPath(false, hash, name); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); @@ -843,11 +882,12 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { if (storePath.empty()) throw; - printError(format("warning: %1%; using cached result") % e.msg()); + warn("warning: %s; using cached result", e.msg()); + result.etag = expectedETag; } } - if (unpack) { + if (request.unpack) { Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); Path unpackedStorePath; @@ -870,14 +910,17 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa } if (expectedStorePath != "" && storePath != expectedStorePath) { - Hash gotHash = unpack - ? hashPath(expectedHash.type, store->toRealPath(storePath)).first - : hashFile(expectedHash.type, store->toRealPath(storePath)); - throw nix::Error("hash mismatch in file downloaded from '%s': got hash '%s' instead of the expected hash '%s'", - url, gotHash.to_string(), expectedHash.to_string()); + unsigned int statusCode = 102; + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + url, request.expectedHash.to_string(), gotHash.to_string()); } - return store->toRealPath(storePath); + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; } diff --git a/src/libstore/download.hh b/src/libstore/download.hh index f0228f7d0..68565bf46 100644 --- a/src/libstore/download.hh +++ b/src/libstore/download.hh @@ -9,13 +9,37 @@ namespace nix { +struct DownloadSettings : Config +{ + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting<unsigned long> stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + + Setting<unsigned int> tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + struct DownloadRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = 5; + size_t tries = downloadSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -36,15 +60,39 @@ struct DownloadResult { bool cached = false; std::string etag; - std::string effectiveUrl; + std::string effectiveUri; std::shared_ptr<std::string> data; uint64_t bodySize = 0; }; +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + +struct CachedDownloadResult +{ + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional<std::string> etag; + std::string effectiveUri; +}; + class Store; struct Downloader { + virtual ~Downloader() { } + /* Enqueue a download request, returning a future to the result of the download. The future may throw a DownloadError exception. */ @@ -64,8 +112,7 @@ struct Downloader and is more recent than ‘tarball-ttl’ seconds. Otherwise, use the recorded ETag to verify if the server has a more recent version, and if so, download it to the Nix store. */ - Path downloadCached(ref<Store> store, const string & uri, bool unpack, string name = "", - const Hash & expectedHash = Hash(), string * effectiveUri = nullptr, int ttl = settings.tarballTtl); + CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index f703e1d15..64780a6da 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -19,6 +19,8 @@ public: uint64_t narOffset = 0; // regular files only }; + virtual ~FSAccessor() { } + virtual Stat stat(const Path & path) = 0; virtual StringSet readDirectory(const Path & path) = 0; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 65220a9f6..7d3da1cc8 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -29,7 +29,7 @@ static string gcRootsDir = "gcroots"; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -int LocalStore::openGCLock(LockType lockType) +AutoCloseFD LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % stateDir % gcLockName).str(); @@ -49,7 +49,7 @@ int LocalStore::openGCLock(LockType lockType) process that can open the file for reading can DoS the collector. */ - return fdGCLock.release(); + return fdGCLock; } @@ -129,8 +129,8 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(); - if (roots.find(gcRoot) == roots.end()) + Roots roots = findRoots(false); + if (roots[storePath].count(gcRoot) == 0) printError( format( "warning: '%1%' is not in a directory where the garbage collector looks for roots; " @@ -197,10 +197,11 @@ void LocalStore::addTempRoot(const Path & path) } -std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) -{ - std::set<std::pair<pid_t, Path>> tempRoots; +static std::string censored = "{censored}"; + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +{ /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { @@ -220,26 +221,22 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); //if (*fd == -1) continue; - if (path != fnTempRoots) { - - /* Try to acquire a write lock without blocking. This can - only succeed if the owning process has died. In that case - we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); - unlink(path.c_str()); - writeFull(fd->get(), "d"); - continue; - } - - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; } + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + /* Read the entire file. */ string contents = readFile(fd->get()); @@ -250,14 +247,12 @@ std::set<std::pair<pid_t, Path>> LocalStore::readTempRoots(FDs & fds) Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); assertStorePath(root); - tempRoots.emplace(pid, root); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } fds.push_back(fd); /* keep open */ } - - return tempRoots; } @@ -266,7 +261,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); else printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); }; @@ -306,7 +301,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) else if (type == DT_REG) { Path storePath = storeDir + "/" + baseNameOf(path); if (isStorePath(storePath) && isValidPath(storePath)) - roots[path] = storePath; + roots[storePath].emplace(path); } } @@ -321,44 +316,31 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } -Roots LocalStore::findRootsNoTemp() +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { - Roots roots; - /* Process direct roots in {gcroots,profiles}. */ findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); - /* Add additional roots returned by the program specified by the - NIX_ROOT_FINDER environment variable. This is typically used - to add running programs to the set of roots (to prevent them - from being garbage collected). */ - size_t n = 0; - for (auto & root : findRuntimeRoots()) - roots[fmt("{memory:%d}", n++)] = root; - - return roots; + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ + findRuntimeRoots(roots, censor); } -Roots LocalStore::findRoots() +Roots LocalStore::findRoots(bool censor) { - Roots roots = findRootsNoTemp(); + Roots roots; + findRootsNoTemp(roots, censor); FDs fds; - pid_t prev = -1; - size_t n = 0; - for (auto & root : readTempRoots(fds)) { - if (prev != root.first) n = 0; - prev = root.first; - roots[fmt("{temp:%d:%d}", root.first, n++)] = root.second; - } + findTempRoots(fds, roots, censor); return roots; } - -static void readProcLink(const string & file, StringSet & paths) +static void readProcLink(const string & file, Roots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -377,8 +359,8 @@ try_again: goto try_again; } if (res > 0 && buf[0] == '/') - paths.emplace(static_cast<char *>(buf), res); - return; + roots[std::string(static_cast<char *>(buf), res)] + .emplace(file); } static string quoteRegexChars(const string & raw) @@ -387,20 +369,20 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, StringSet & paths) +static void readFileRoots(const char * path, Roots & roots) { try { - paths.emplace(readFile(path)); + roots[readFile(path)].emplace(path); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; } } -PathSet LocalStore::findRuntimeRoots() +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - PathSet roots; - StringSet paths; + Roots unchecked; + auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; @@ -410,10 +392,10 @@ PathSet LocalStore::findRuntimeRoots() while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink((format("/proc/%1%/exe") % ent->d_name).str(), paths); - readProcLink((format("/proc/%1%/cwd") % ent->d_name).str(), paths); + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - auto fdStr = (format("/proc/%1%/fd") % ent->d_name).str(); + auto fdStr = fmt("/proc/%s/fd", ent->d_name); auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); if (!fdDir) { if (errno == ENOENT || errno == EACCES) @@ -422,9 +404,8 @@ PathSet LocalStore::findRuntimeRoots() } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') { - readProcLink((format("%1%/%2%") % fdStr % fd_ent->d_name).str(), paths); - } + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); } if (errno) { if (errno == ESRCH) @@ -434,18 +415,19 @@ PathSet LocalStore::findRuntimeRoots() fdDir.reset(); try { - auto mapLines = - tokenizeString<std::vector<string>>(readFile((format("/proc/%1%/maps") % ent->d_name).str(), true), "\n"); - for (const auto& line : mapLines) { + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); + for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) - paths.emplace(match[1]); + unchecked[match[1]].emplace(mapFile); } - auto envString = readFile((format("/proc/%1%/environ") % ent->d_name).str(), true); + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile, true); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) - paths.emplace(i->str()); + unchecked[i->str()].emplace(envFile); } catch (SysError & e) { if (errno == ENOENT || errno == EACCES || errno == ESRCH) continue; @@ -458,36 +440,43 @@ PathSet LocalStore::findRuntimeRoots() } #if !defined(__linux__) - try { - std::regex lsofRegex(R"(^n(/.*)$)"); - auto lsofLines = - tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); - for (const auto & line : lsofLines) { - std::smatch match; - if (std::regex_match(line, match, lsofRegex)) - paths.emplace(match[1]); + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ } - } catch (ExecError & e) { - /* lsof not installed, lsof failed */ } #endif #if defined(__linux__) - readFileRoots("/proc/sys/kernel/modprobe", paths); - readFileRoots("/proc/sys/kernel/fbsplash", paths); - readFileRoots("/proc/sys/kernel/poweroff_cmd", paths); + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); #endif - for (auto & i : paths) - if (isInStore(i)) { - Path path = toStorePath(i); - if (roots.find(path) == roots.end() && isStorePath(path) && isValidPath(path)) { + for (auto & [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); + if (isStorePath(path) && isValidPath(path)) { debug(format("got additional root '%1%'") % path); - roots.insert(path); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); } } - - return roots; + } } @@ -701,9 +690,8 @@ void LocalStore::removeUnusedLinks(const GCState & state) throw SysError(format("statting '%1%'") % path); if (st.st_nlink != 1) { - unsigned long long size = st.st_blocks * 512ULL; - actualSize += size; - unsharedSize += (st.st_nlink - 1) * size; + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; continue; } @@ -712,7 +700,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) if (unlink(path.c_str()) == -1) throw SysError(format("deleting '%1%'") % path); - state.results.bytesFreed += st.st_blocks * 512ULL; + state.results.bytesFreed += st.st_size; } struct stat st; @@ -754,16 +742,20 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printError(format("finding garbage collector roots...")); - Roots rootMap = options.ignoreLiveness ? Roots() : findRootsNoTemp(); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.second); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths can be added to the set of temporary roots. */ FDs fds; - for (auto & root : readTempRoots(fds)) - state.tempRoots.insert(root.second); + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto & root : tempRoots) + state.tempRoots.insert(root.first); state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); /* After this point the set of roots or temporary roots cannot @@ -878,7 +870,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { - auto getAvail = [this]() { + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + struct statvfs st; if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); @@ -899,7 +896,7 @@ void LocalStore::autoGC(bool sync) auto now = std::chrono::steady_clock::now(); - if (now < state->lastGCCheck + std::chrono::seconds(5)) return; + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; auto avail = getAvail(); @@ -926,11 +923,11 @@ void LocalStore::autoGC(bool sync) promise.set_value(); }); - printInfo("running auto-GC to free %d bytes", settings.maxFree - avail); - GCOptions options; options.maxFreed = settings.maxFree - avail; + printInfo("running auto-GC to free %d bytes", options.maxFreed); + GCResults results; collectGarbage(options, results); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d95db5672..1c2c08715 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -78,7 +78,11 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); - globalConfig.applyConfigFile(getConfigDir() + "/nix/nix.conf"); + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } } unsigned int Settings::getDefaultCores() @@ -86,6 +90,21 @@ unsigned int Settings::getDefaultCores() return std::max(1U, std::thread::hardware_concurrency()); } +StringSet Settings::getDefaultSystemFeatures() +{ + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel"}; + + #if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) + features.insert("kvm"); + #endif + + return features; +} + const string nixVersion = PACKAGE_VERSION; template<> void BaseSetting<SandboxMode>::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index f589078db..ab1c09aa2 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -32,6 +32,8 @@ class Settings : public Config { unsigned int getDefaultCores(); + StringSet getDefaultSystemFeatures(); + public: Settings(); @@ -80,9 +82,9 @@ public: /* Whether to show build log output in real time. */ bool verboseBuild = true; - /* If verboseBuild is false, the number of lines of the tail of - the log to show if a build fails. */ - size_t logLines = 10; + Setting<size_t> logLines{this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", "Maximum number of parallel build jobs. \"auto\" means use number of cores.", @@ -193,7 +195,13 @@ public: Setting<bool> showTrace{this, false, "show-trace", "Whether to show a stack trace on evaluation errors."}; - Setting<SandboxMode> sandboxMode{this, smDisabled, "sandbox", + Setting<SandboxMode> sandboxMode{this, + #if __linux__ + smEnabled + #else + smDisabled + #endif + , "sandbox", "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", {"build-use-chroot", "build-use-sandbox"}}; @@ -201,6 +209,9 @@ public: "The paths to make available inside the build sandbox.", {"build-chroot-dirs", "build-sandbox-paths"}}; + Setting<bool> sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting<PathSet> extraSandboxPaths{this, {}, "extra-sandbox-paths", "Additional paths to make available inside the build sandbox.", {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; @@ -247,7 +258,7 @@ public: "Secret keys with which to sign local builds."}; Setting<unsigned int> tarballTtl{this, 60 * 60, "tarball-ttl", - "How soon to expire files fetched by builtins.fetchTarball and builtins.fetchurl."}; + "How long downloaded files are considered up-to-date."}; Setting<bool> requireSigs{this, true, "require-sigs", "Whether to check that any non-content-addressed path added to the " @@ -261,6 +272,10 @@ public: "These may be supported natively (e.g. armv7 on some aarch64 CPUs " "or using hacks like qemu-user."}; + Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), + "system-features", + "Optional features that this system implements (like \"kvm\")."}; + Setting<Strings> substituters{this, nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), "substituters", @@ -303,6 +318,9 @@ public: "pre-build-hook", "A program to run just before a build to set derivation-specific build settings."}; + Setting<std::string> postBuildHook{this, "", "post-build-hook", + "A program to run just after each succesful build."}; + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", "Path to the netrc file used to obtain usernames/passwords for downloads."}; @@ -330,6 +348,9 @@ public: Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), "max-free", "Stop deleting garbage when free disk space is above the specified amount."}; + Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index ab524d523..779f89e68 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -13,6 +13,14 @@ private: Path cacheUri; + struct State + { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync<State> _state; + public: HttpBinaryCacheStore( @@ -46,12 +54,36 @@ public: protected: + void maybeDisable() + { + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", getUri(), t); + state->enabled = false; + state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); + } + } + + void checkEnabled() + { + auto state(_state.lock()); + if (state->enabled) return; + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", getUri()); + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + bool fileExists(const std::string & path) override { + checkEnabled(); + try { DownloadRequest request(cacheUri + "/" + path); request.head = true; - request.tries = 5; getDownloader()->download(request); return true; } catch (DownloadError & e) { @@ -59,6 +91,7 @@ protected: bucket is unlistable, so treat 403 as 404. */ if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) return false; + maybeDisable(); throw; } } @@ -80,37 +113,43 @@ protected: DownloadRequest makeRequest(const std::string & path) { DownloadRequest request(cacheUri + "/" + path); - request.tries = 8; return request; } void getFile(const std::string & path, Sink & sink) override { + checkEnabled(); auto request(makeRequest(path)); try { getDownloader()->download(std::move(request), sink); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + maybeDisable(); throw; } } void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) override + Callback<std::shared_ptr<std::string>> callback) noexcept override { + checkEnabled(); + auto request(makeRequest(path)); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + getDownloader()->enqueueDownload(request, - {[callback](std::future<DownloadResult> result) { + {[callbackPtr, this](std::future<DownloadResult> result) { try { - callback(result.get().data); + (*callbackPtr)(result.get().data); } catch (DownloadError & e) { if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) - return callback(std::shared_ptr<std::string>()); - callback.rethrow(); + return (*callbackPtr)(std::shared_ptr<std::string>()); + maybeDisable(); + callbackPtr->rethrow(); } catch (...) { - callback.rethrow(); + callbackPtr->rethrow(); } }}); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7c214f09d..d5fbdd25a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -88,7 +88,7 @@ struct LegacySSHStore : public Store } void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override { try { auto conn(connections->get()); @@ -131,7 +131,7 @@ struct LegacySSHStore : public Store auto conn(connections->get()); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { conn->to << cmdAddToStoreNar @@ -187,28 +187,17 @@ struct LegacySSHStore : public Store copyNAR(conn->from, sink); } - PathSet queryAllValidPaths() override { unsupported(); } - - void queryReferrers(const Path & path, PathSet & referrers) override - { unsupported(); } - - PathSet queryDerivationOutputs(const Path & path) override - { unsupported(); } - - StringSet queryDerivationOutputNames(const Path & path) override - { unsupported(); } - Path queryPathFromHashPart(const string & hashPart) override - { unsupported(); } + { unsupported("queryPathFromHashPart"); } Path addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override - { unsupported(); } + { unsupported("addToStore"); } Path addTextToStore(const string & name, const string & s, const PathSet & references, RepairFlag repair) override - { unsupported(); } + { unsupported("addTextToStore"); } BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) override @@ -242,25 +231,7 @@ struct LegacySSHStore : public Store } void ensurePath(const Path & path) override - { unsupported(); } - - void addTempRoot(const Path & path) override - { unsupported(); } - - void addIndirectRoot(const Path & path) override - { unsupported(); } - - Roots findRoots() override - { unsupported(); } - - void collectGarbage(const GCOptions & options, GCResults & results) override - { unsupported(); } - - ref<FSAccessor> getFSAccessor() override - { unsupported(); } - - void addSignatures(const Path & storePath, const StringSet & sigs) override - { unsupported(); } + { unsupported("ensurePath"); } void computeFSClosure(const PathSet & paths, PathSet & out, bool flipDirection = false, @@ -303,6 +274,12 @@ struct LegacySSHStore : public Store { auto conn(connections->get()); } + + unsigned int getProtocol() override + { + auto conn(connections->get()); + return conn->remoteVersion; + } }; static RegisterStoreImplementation regStore([]( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index b7001795b..9f99ee76d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -63,6 +63,8 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + if (writeDebugInfo) + createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3b2ba65f3..a2af51d0e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -450,7 +450,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); if (eaSize < 0) { - if (errno != ENOTSUP) + if (errno != ENOTSUP && errno != ENODATA) throw SysError("querying extended attributes of '%s'", path); } else if (eaSize > 0) { std::vector<char> eaBuf(eaSize); @@ -629,7 +629,7 @@ uint64_t LocalStore::addValidPath(State & state, void LocalStore::queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { try { auto info = std::make_shared<ValidPathInfo>(); @@ -879,7 +879,13 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, info->references, narInfo ? narInfo->fileSize : 0, info->narSize}; - } catch (InvalidPath) { + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + printError(e.what()); + else + throw; } } } @@ -1014,11 +1020,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto hashResult = hashSink.finish(); if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s'; expected hash '%s', got '%s'", + throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narHash.to_string(), hashResult.first.to_string()); if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s'; expected %s, got %s", + throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", info.path, info.narSize, hashResult.second); autoGC(); @@ -1204,7 +1210,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) bool errors = false; - /* Acquire the global GC lock to prevent a garbage collection. */ + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); PathSet store; @@ -1215,13 +1222,11 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + fdGCLock = -1; + for (auto & i : validPaths2) verifyPath(i, store, done, validPaths, repair, errors); - /* Release the GC lock so that checking content hashes (which can - take ages) doesn't block the GC or builds. */ - fdGCLock = -1; - /* Optionally, check the content hashes (slow). */ if (checkContents) { printInfo("checking hashes..."); @@ -1332,6 +1337,12 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, } +unsigned int LocalStore::getProtocol() +{ + return PROTOCOL_VERSION; +} + + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) static void makeMutable(const Path & path) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 746bdbeed..3ae34c403 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; @@ -180,11 +180,11 @@ private: typedef std::shared_ptr<AutoCloseFD> FDPtr; typedef list<FDPtr> FDs; - std::set<std::pair<pid_t, Path>> readTempRoots(FDs & fds); + void findTempRoots(FDs & fds, Roots & roots, bool censor); public: - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -209,6 +209,8 @@ public: void registerValidPaths(const ValidPathInfos & infos); + unsigned int getProtocol() override; + void vacuumDB(); /* Repair the contents of the given path by redownloading it using @@ -261,13 +263,13 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); - int openGCLock(LockType lockType); + AutoCloseFD openGCLock(LockType lockType); void findRoots(const Path & path, unsigned char type, Roots & roots); - Roots findRootsNoTemp(); + void findRootsNoTemp(Roots & roots, bool censor); - PathSet findRuntimeRoots(); + void findRuntimeRoots(Roots & roots, bool censor); void removeUnusedLinks(const GCState & state); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 3799257f8..89fc918c3 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libformat +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index edd03d147..f848582da 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -89,10 +89,11 @@ void parseMachines(const std::string & s, Machines & machines) Machines getMachines() { - Machines machines; - - parseMachines(settings.builders, machines); - + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); return machines; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index adcce026f..dddf13430 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,4 +1,5 @@ #include "derivations.hh" +#include "parsed-derivations.hh" #include "globals.hh" #include "local-store.hh" #include "store-api.hh" @@ -189,6 +190,7 @@ void Store::queryMissing(const PathSet & targets, } Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); PathSet invalid; for (auto & j : drv.outputs) @@ -197,7 +199,7 @@ void Store::queryMissing(const PathSet & targets, invalid.insert(j.second.path); if (invalid.empty()) return; - if (settings.useSubstitutes && drv.substitutesAllowed()) { + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 35403e5df..32ad7f2b2 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -31,6 +31,7 @@ create table if not exists NARs ( refs text, deriver text, sigs text, + ca text, timestamp integer not null, present integer not null, primary key (cache, hashPart), @@ -72,7 +73,7 @@ public: { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite"; + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -94,13 +95,13 @@ public: state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " - "narSize, refs, deriver, sigs, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); state->insertMissingNAR.create(state->db, "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); state->queryNAR.create(state->db, - "select * from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); /* Periodically purge expired entries from the database. */ retrySQLite<void>([&]() { @@ -189,27 +190,28 @@ public: if (!queryNAR.next()) return {oUnknown, 0}; - if (!queryNAR.getInt(13)) + if (!queryNAR.getInt(0)) return {oInvalid, 0}; auto narInfo = make_ref<NarInfo>(); - auto namePart = queryNAR.getStr(2); + auto namePart = queryNAR.getStr(1); narInfo->path = cache.storeDir + "/" + hashPart + (namePart.empty() ? "" : "-" + namePart); - narInfo->url = queryNAR.getStr(3); - narInfo->compression = queryNAR.getStr(4); - if (!queryNAR.isNull(5)) - narInfo->fileHash = Hash(queryNAR.getStr(5)); - narInfo->fileSize = queryNAR.getInt(6); - narInfo->narHash = Hash(queryNAR.getStr(7)); - narInfo->narSize = queryNAR.getInt(8); - for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " ")) + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) + narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) narInfo->references.insert(cache.storeDir + "/" + r); - if (!queryNAR.isNull(10)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(10); - for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " ")) + if (!queryNAR.isNull(9)) + narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); + narInfo->ca = queryNAR.getStr(11); return {oValid, narInfo}; }); @@ -243,6 +245,7 @@ public: (concatStringsSep(" ", info->shortRefs())) (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") (concatStringsSep(" ", info->sigs)) + (info->ca) (time(0)).exec(); } else { diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 5cf22faad..6d67b1e03 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++14 +Cflags: -I${includedir}/nix -std=c++17 diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc new file mode 100644 index 000000000..87be8a24e --- /dev/null +++ b/src/libstore/parsed-derivations.cc @@ -0,0 +1,116 @@ +#include "parsed-derivations.hh" + +namespace nix { + +ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) +{ + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = nlohmann::json::parse(jsonAttr->second); + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + } + } +} + +std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); + return i->get<std::string>(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return i->second; + } +} + +bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); + return i->get<bool>(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return def; + else + return i->second == "1"; + } +} + +std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get<std::string>()); + } + return res; + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return tokenizeString<Strings>(i->second); + } +} + +StringSet ParsedDerivation::getRequiredSystemFeatures() const +{ + StringSet res; + for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) + res.insert(i); + return res; +} + +bool ParsedDerivation::canBuildLocally() const +{ + if (drv.platform != settings.thisSystem.get() + && !settings.extraPlatforms.get().count(drv.platform) + && !drv.isBuiltin()) + return false; + + for (auto & feature : getRequiredSystemFeatures()) + if (!settings.systemFeatures.get().count(feature)) return false; + + return true; +} + +bool ParsedDerivation::willBuildLocally() const +{ + return getBoolAttr("preferLocalBuild") && canBuildLocally(); +} + +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + +} diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh new file mode 100644 index 000000000..9bde4b4dc --- /dev/null +++ b/src/libstore/parsed-derivations.hh @@ -0,0 +1,37 @@ +#include "derivations.hh" + +#include <nlohmann/json.hpp> + +namespace nix { + +class ParsedDerivation +{ + Path drvPath; + BasicDerivation & drv; + std::optional<nlohmann::json> structuredAttrs; + +public: + + ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + + const std::optional<nlohmann::json> & getStructuredAttrs() const + { + return structuredAttrs; + } + + std::optional<std::string> getStringAttr(const std::string & name) const; + + bool getBoolAttr(const std::string & name, bool def = false) const; + + std::optional<Strings> getStringsAttr(const std::string & name) const; + + StringSet getRequiredSystemFeatures() const; + + bool canBuildLocally() const; + + bool willBuildLocally() const; + + bool substitutesAllowed() const; +}; + +} diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 08d1efdbe..2635e3940 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -5,9 +5,10 @@ #include <cerrno> #include <cstdlib> +#include <fcntl.h> #include <sys/types.h> #include <sys/stat.h> -#include <fcntl.h> +#include <sys/file.h> namespace nix { @@ -40,17 +41,14 @@ void deleteLockFile(const Path & path, int fd) bool lockFile(int fd, LockType lockType, bool wait) { - struct flock lock; - if (lockType == ltRead) lock.l_type = F_RDLCK; - else if (lockType == ltWrite) lock.l_type = F_WRLCK; - else if (lockType == ltNone) lock.l_type = F_UNLCK; + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; else abort(); - lock.l_whence = SEEK_SET; - lock.l_start = 0; - lock.l_len = 0; /* entire file */ if (wait) { - while (fcntl(fd, F_SETLKW, &lock) != 0) { + while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); @@ -58,9 +56,9 @@ bool lockFile(int fd, LockType lockType, bool wait) return false; } } else { - while (fcntl(fd, F_SETLK, &lock) != 0) { + while (flock(fd, type | LOCK_NB) != 0) { checkInterrupt(); - if (errno == EACCES || errno == EAGAIN) return false; + if (errno == EWOULDBLOCK) return false; if (errno != EINTR) throw SysError(format("acquiring/releasing lock")); } @@ -70,14 +68,6 @@ bool lockFile(int fd, LockType lockType, bool wait) } -/* This enables us to check whether are not already holding a lock on - a file ourselves. POSIX locks (fcntl) suck in this respect: if we - close a descriptor, the previous lock will be closed as well. And - there is no way to query whether we already have a lock (F_GETLK - only works on locks held by other processes). */ -static Sync<StringSet> lockedPaths_; - - PathLocks::PathLocks() : deletePaths(false) { @@ -91,7 +81,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) } -bool PathLocks::lockPaths(const PathSet & _paths, +bool PathLocks::lockPaths(const PathSet & paths, const string & waitMsg, bool wait) { assert(fds.empty()); @@ -99,75 +89,54 @@ bool PathLocks::lockPaths(const PathSet & _paths, /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ - /* Sort the paths. This assures that locks are always acquired in - the same order, thus preventing deadlocks. */ - Paths paths(_paths.begin(), _paths.end()); - paths.sort(); - - /* Acquire the lock for each path. */ + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ for (auto & path : paths) { checkInterrupt(); Path lockPath = path + ".lock"; debug(format("locking path '%1%'") % path); - { - auto lockedPaths(lockedPaths_.lock()); - if (lockedPaths->count(lockPath)) { - if (!wait) return false; - throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath); - } - lockedPaths->insert(lockPath); - } - - try { - - AutoCloseFD fd; + AutoCloseFD fd; - while (1) { + while (1) { - /* Open/create the lock file. */ - fd = openLockFile(lockPath, true); + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); - /* Acquire an exclusive lock. */ - if (!lockFile(fd.get(), ltWrite, false)) { - if (wait) { - if (waitMsg != "") printError(waitMsg); - lockFile(fd.get(), ltWrite, true); - } else { - /* Failed to lock this path; release all other - locks. */ - unlock(); - lockedPaths_.lock()->erase(lockPath); - return false; - } + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; } - - debug(format("lock acquired on '%1%'") % lockPath); - - /* Check that the lock file hasn't become stale (i.e., - hasn't been unlinked). */ - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); - if (st.st_size != 0) - /* This lock file has been unlinked, so we're holding - a lock on a deleted file. This means that other - processes may create and acquire a lock on - `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); - else - break; } - /* Use borrow so that the descriptor isn't closed. */ - fds.push_back(FDPair(fd.release(), lockPath)); - - } catch (...) { - lockedPaths_.lock()->erase(lockPath); - throw; + debug(format("lock acquired on '%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; } + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); } return true; @@ -189,8 +158,6 @@ void PathLocks::unlock() for (auto & i : fds) { if (deletePaths) deleteLockFile(i.second, i.first); - lockedPaths_.lock()->erase(i.second); - if (close(i.first) == -1) printError( format("error (ignored): cannot close lock file on '%1%'") % i.second); @@ -208,11 +175,4 @@ void PathLocks::setDeletion(bool deletePaths) } -bool pathIsLockedByMe(const Path & path) -{ - Path lockPath = path + ".lock"; - return lockedPaths_.lock()->count(lockPath); -} - - } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index db51f950a..411da0222 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -16,8 +16,6 @@ enum LockType { ltRead, ltWrite, ltNone }; bool lockFile(int fd, LockType lockType, bool wait); -MakeError(AlreadyLocked, Error); - class PathLocks { private: @@ -37,6 +35,4 @@ public: void setDeletion(bool deletePaths); }; -bool pathIsLockedByMe(const Path & path); - } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ea86ef052..b9e7a80ba 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -161,7 +161,8 @@ void RemoteStore::initConnection(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) conn.to << false; - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); } catch (Error & e) { throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); @@ -190,27 +191,81 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map<std::string, Config::SettingInfo> overrides; globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; } - conn.processStderr(); + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); +} + + +/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle +{ + Pool<RemoteStore::Connection>::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0) + { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); } bool RemoteStore::isValidPathUncached(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopIsValidPath << path; - conn->processStderr(); + conn.processStderr(); return readInt(conn->from); } PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) @@ -218,7 +273,7 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe return res; } else { conn->to << wopQueryValidPaths << paths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } } @@ -226,27 +281,27 @@ PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybe PathSet RemoteStore::queryAllValidPaths() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryAllValidPaths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { PathSet res; for (auto & i : paths) { conn->to << wopHasSubstitutes << i; - conn->processStderr(); + conn.processStderr(); if (readInt(conn->from)) res.insert(i); } return res; } else { conn->to << wopQuerySubstitutablePaths << paths; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } } @@ -257,14 +312,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, { if (paths.empty()) return; - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { for (auto & i : paths) { SubstitutablePathInfo info; conn->to << wopQuerySubstitutablePathInfo << i; - conn->processStderr(); + conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; info.deriver = readString(conn->from); @@ -278,7 +333,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } else { conn->to << wopQuerySubstitutablePathInfos << paths; - conn->processStderr(); + conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { Path path = readStorePath(*this, conn->from); @@ -295,15 +350,15 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, void RemoteStore::queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { try { std::shared_ptr<ValidPathInfo> info; { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryPathInfo << path; try { - conn->processStderr(); + conn.processStderr(); } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) @@ -335,9 +390,9 @@ void RemoteStore::queryPathInfoUncached(const Path & path, void RemoteStore::queryReferrers(const Path & path, PathSet & referrers) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryReferrers << path; - conn->processStderr(); + conn.processStderr(); PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); referrers.insert(referrers2.begin(), referrers2.end()); } @@ -345,36 +400,36 @@ void RemoteStore::queryReferrers(const Path & path, PathSet RemoteStore::queryValidDerivers(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryValidDerivers << path; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << path; - conn->processStderr(); + conn.processStderr(); return readStorePaths<PathSet>(*this, conn->from); } PathSet RemoteStore::queryDerivationOutputNames(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryDerivationOutputNames << path; - conn->processStderr(); + conn.processStderr(); return readStrings<PathSet>(conn->from); } Path RemoteStore::queryPathFromHashPart(const string & hashPart) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopQueryPathFromHashPart << hashPart; - conn->processStderr(); + conn.processStderr(); Path path = readString(conn->from); if (!path.empty()) assertStorePath(path); return path; @@ -384,7 +439,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { conn->to << wopImportPaths; @@ -403,7 +458,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, ; }); - conn->processStderr(0, source2.get()); + conn.processStderr(0, source2.get()); auto importedPaths = readStorePaths<PathSet>(*this, conn->from); assert(importedPaths.size() <= 1); @@ -417,7 +472,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); - conn->processStderr(0, tunnel ? &source : nullptr); + conn.processStderr(0, tunnel ? &source : nullptr); } } @@ -427,7 +482,7 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); Path srcPath(absPath(_srcPath)); @@ -445,13 +500,13 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, dumpPath(srcPath, conn->to, filter); } conn->to.warn = false; - conn->processStderr(); + conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM or I/O error. */ if (e.errNo == EPIPE) try { - conn->processStderr(); + conn.processStderr(); } catch (EndOfFile & e) { } throw; } @@ -465,17 +520,17 @@ Path RemoteStore::addTextToStore(const string & name, const string & s, { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddTextToStore << name << s << references; - conn->processStderr(); + conn.processStderr(); return readStorePath(*this, conn->from); } void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildPaths; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { conn->to << drvPaths; @@ -494,7 +549,7 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) drvPaths2.insert(string(i, 0, i.find('!'))); conn->to << drvPaths2; } - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } @@ -502,9 +557,9 @@ void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopBuildDerivation << drvPath << drv << buildMode; - conn->processStderr(); + conn.processStderr(); BuildResult res; unsigned int status; conn->from >> status >> res.errorMsg; @@ -515,51 +570,51 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva void RemoteStore::ensurePath(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopEnsurePath << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addTempRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddTempRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::addIndirectRoot(const Path & path) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddIndirectRoot << path; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } void RemoteStore::syncWithGC() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopSyncWithGC; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } -Roots RemoteStore::findRoots() +Roots RemoteStore::findRoots(bool censor) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopFindRoots; - conn->processStderr(); + conn.processStderr(); size_t count = readNum<size_t>(conn->from); Roots result; while (count--) { Path link = readString(conn->from); Path target = readStorePath(*this, conn->from); - result[link] = target; + result[target].emplace(link); } return result; } @@ -567,7 +622,7 @@ Roots RemoteStore::findRoots() void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness @@ -575,7 +630,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) /* removed options */ << 0 << 0 << 0; - conn->processStderr(); + conn.processStderr(); results.paths = readStrings<PathSet>(conn->from); results.bytesFreed = readLongLong(conn->from); @@ -590,27 +645,27 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopOptimiseStore; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopVerifyStore << checkContents << repair; - conn->processStderr(); + conn.processStderr(); return readInt(conn->from); } void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) { - auto conn(connections->get()); + auto conn(getConnection()); conn->to << wopAddSignatures << storePath << sigs; - conn->processStderr(); + conn.processStderr(); readInt(conn->from); } @@ -620,13 +675,13 @@ void RemoteStore::queryMissing(const PathSet & targets, unsigned long long & downloadSize, unsigned long long & narSize) { { - auto conn(connections->get()); + auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; conn->to << wopQueryMissing << targets; - conn->processStderr(); + conn.processStderr(); willBuild = readStorePaths<PathSet>(*this, conn->from); willSubstitute = readStorePaths<PathSet>(*this, conn->from); unknown = readStorePaths<PathSet>(*this, conn->from); @@ -642,7 +697,14 @@ void RemoteStore::queryMissing(const PathSet & targets, void RemoteStore::connect() { + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ auto conn(connections->get()); + return conn->daemonVersion; } @@ -679,7 +741,7 @@ static Logger::Fields readFields(Source & from) } -void RemoteStore::Connection::processStderr(Sink * sink, Source * source) +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) { to.flush(); @@ -704,7 +766,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else if (msg == STDERR_ERROR) { string error = readString(from); unsigned int status = readInt(from); - throw Error(status, error); + return std::make_exception_ptr(Error(status, error)); } else if (msg == STDERR_NEXT) @@ -738,6 +800,8 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source) else throw Error("got unknown message type %x from Nix daemon", msg); } + + return nullptr; } static std::string uriScheme = "unix://"; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b488e34ce..82fbec092 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -14,6 +14,7 @@ class Pid; struct FdSink; struct FdSource; template<typename T> class Pool; +struct ConnectionHandle; /* FIXME: RemoteStore is a misnomer - should be something like @@ -40,7 +41,7 @@ public: PathSet queryAllValidPaths() override; void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) override; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; void queryReferrers(const Path & path, PathSet & referrers) override; @@ -81,7 +82,7 @@ public: void syncWithGC() override; - Roots findRoots() override; + Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -97,12 +98,15 @@ public: void connect() override; + unsigned int getProtocol() override; + void flushBadConnections(); protected: struct Connection { + AutoCloseFD fd; FdSink to; FdSource from; unsigned int daemonVersion; @@ -110,7 +114,7 @@ protected: virtual ~Connection(); - void processStderr(Sink * sink = 0, Source * source = 0); + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); }; ref<Connection> openConnectionWrapper(); @@ -123,6 +127,10 @@ protected: virtual void setOptions(Connection & conn); + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + private: std::atomic_bool failed{false}; @@ -140,13 +148,8 @@ public: private: - struct Connection : RemoteStore::Connection - { - AutoCloseFD fd; - }; - ref<RemoteStore::Connection> openConnection() override; - std::experimental::optional<std::string> path; + std::optional<std::string> path; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 6d95c1fa8..cd547a964 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -19,8 +19,6 @@ #include <aws/core/utils/logging/LogMacros.h> #include <aws/core/utils/threading/Executor.h> #include <aws/s3/S3Client.h> -#include <aws/s3/model/CreateBucketRequest.h> -#include <aws/s3/model/GetBucketLocationRequest.h> #include <aws/s3/model/GetObjectRequest.h> #include <aws/s3/model/HeadObjectRequest.h> #include <aws/s3/model/ListObjectsRequest.h> @@ -84,8 +82,8 @@ static void initAWS() }); } -S3Helper::S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint) - : config(makeConfig(region, endpoint)) +S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) + : config(makeConfig(region, scheme, endpoint)) , client(make_ref<Aws::S3::S3Client>( profile == "" ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( @@ -116,15 +114,19 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & endpoint) +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) { initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } if (!endpoint.empty()) { res->endpointOverride = endpoint; } res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; res->retryStrategy = std::make_shared<RetryStrategy>(); res->caFile = settings.caFile; return res; @@ -171,10 +173,13 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + const Setting<bool> multipartUpload{ + this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting<uint64_t> bufferSize{ this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; @@ -188,7 +193,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore const Params & params, const std::string & bucketName) : S3BinaryCacheStore(params) , bucketName(bucketName) - , s3Helper(profile, region, endpoint) + , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); } @@ -202,32 +207,6 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { - /* Create the bucket if it doesn't already exists. */ - // FIXME: HeadBucket would be more appropriate, but doesn't return - // an easily parsed 404 message. - auto res = s3Helper.client->GetBucketLocation( - Aws::S3::Model::GetBucketLocationRequest().WithBucket(bucketName)); - - if (!res.IsSuccess()) { - if (res.GetError().GetErrorType() != Aws::S3::S3Errors::NO_SUCH_BUCKET) - throw Error(format("AWS error checking bucket '%s': %s") % bucketName % res.GetError().GetMessage()); - - printInfo("creating S3 bucket '%s'...", bucketName); - - // Stupid S3 bucket locations. - auto bucketConfig = Aws::S3::Model::CreateBucketConfiguration(); - if (s3Helper.config->region != "us-east-1") - bucketConfig.SetLocationConstraint( - Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName( - s3Helper.config->region)); - - checkAws(format("AWS error creating bucket '%s'") % bucketName, - s3Helper.client->CreateBucket( - Aws::S3::Model::CreateBucketRequest() - .WithBucket(bucketName) - .WithCreateBucketConfiguration(bucketConfig))); - } - BinaryCacheStore::init(); diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); @@ -275,6 +254,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return true; } + std::shared_ptr<TransferManager> transferManager; + std::once_flag transferManagerCreated; + void uploadFile(const std::string & path, const std::string & data, const std::string & mimeType, const std::string & contentEncoding) @@ -286,58 +268,73 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> executor = std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>(maxThreads); - TransferManagerConfiguration transferConfig(executor.get()); - - transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; - - transferConfig.uploadProgressCallback = - [&](const TransferManager *transferManager, - const std::shared_ptr<const TransferHandle> - &transferHandle) { - //FIXME: find a way to properly abort the multipart upload. - checkInterrupt(); - debug("upload progress ('%s'): '%d' of '%d' bytes", - path, - transferHandle->GetBytesTransferred(), - transferHandle->GetBytesTotalSize()); - }; + std::call_once(transferManagerCreated, [&]() + { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager *transferManager, + const std::shared_ptr<const TransferHandle> + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); - transferConfig.transferStatusUpdatedCallback = - [&](const TransferManager *, - const std::shared_ptr<const TransferHandle> - &transferHandle) { - switch (transferHandle->GetStatus()) { - case TransferStatus::COMPLETED: - printTalkative("upload of '%s' completed", path); - stats.put++; - stats.putBytes += data.size(); - break; - case TransferStatus::IN_PROGRESS: - break; - case TransferStatus::FAILED: - throw Error("AWS error: failed to upload 's3://%s/%s'", - bucketName, path); - break; - default: - throw Error("AWS error: transfer status of 's3://%s/%s' " - "in unexpected state", - bucketName, path); - }; - }; + auto now1 = std::chrono::steady_clock::now(); - std::shared_ptr<TransferManager> transferManager = - TransferManager::Create(transferConfig); + if (transferManager) { - auto now1 = std::chrono::steady_clock::now(); + if (contentEncoding != "") + throw Error("setting a content encoding is not supported with S3 multi-part uploads"); + + std::shared_ptr<TransferHandle> transferHandle = + transferManager->UploadFile( + stream, bucketName, path, mimeType, + Aws::Map<Aws::String, Aws::String>(), + nullptr /*, contentEncoding */); - std::shared_ptr<TransferHandle> transferHandle = - transferManager->UploadFile( - stream, bucketName, path, mimeType, - Aws::Map<Aws::String, Aws::String>(), - nullptr, contentEncoding); + transferHandle->WaitUntilFinished(); - transferHandle->WaitUntilFinished(); + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", + bucketName, path, transferHandle->GetLastError().GetMessage()); + + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); + + } else { + + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetContentType(mimeType); + + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); + + auto stream = std::make_shared<istringstream_nocopy>(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } auto now2 = std::chrono::steady_clock::now(); @@ -349,6 +346,8 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore bucketName % path % data.size() % duration); stats.putTimeMs += duration; + stats.putBytes += data.size(); + stats.put++; } void upsertFile(const std::string & path, const std::string & data, diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 95d612b66..ef5f23d0f 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -14,9 +14,9 @@ struct S3Helper ref<Aws::Client::ClientConfiguration> config; ref<Aws::S3::S3Client> client; - S3Helper(const std::string & profile, const std::string & region, const std::string & endpoint); + S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); - ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & endpoint); + ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); struct DownloadResult { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7a4a5f5eb..5f63c53b5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -85,18 +85,29 @@ string storePathToHash(const Path & path) void checkStoreName(const string & name) { string validChars = "+-._?="; + + auto baseError = format("The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % validChars % name; + /* Disallow names starting with a dot for possible security reasons (e.g., "." and ".."). */ if (string(name, 0, 1) == ".") - throw Error(format("illegal name: '%1%'") % name); + throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); for (auto & i : name) if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || validChars.find(i) != string::npos)) { - throw Error(format("invalid character '%1%' in name '%2%'") - % i % name); + throw Error(baseError % (format("the '%1%' character is invalid") % i)); } } @@ -318,11 +329,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) void Store::queryPathInfo(const Path & storePath, - Callback<ref<ValidPathInfo>> callback) + Callback<ref<ValidPathInfo>> callback) noexcept { - auto hashPart = storePathToHash(storePath); + std::string hashPart; try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); { auto res = state.lock()->pathInfoCache.get(hashPart); @@ -352,8 +366,10 @@ void Store::queryPathInfo(const Path & storePath, } catch (...) { return callback.rethrow(); } + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callback](std::future<std::shared_ptr<ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -373,8 +389,8 @@ void Store::queryPathInfo(const Path & storePath, throw InvalidPath("path '%s' is not valid", storePath); } - callback(ref<ValidPathInfo>(info)); - } catch (...) { callback.rethrow(); } + (*callbackPtr)(ref<ValidPathInfo>(info)); + } catch (...) { callbackPtr->rethrow(); } }}); } @@ -560,10 +576,10 @@ void Store::buildPaths(const PathSet & paths, BuildMode buildMode) { for (auto & path : paths) if (isDerivation(path)) - unsupported(); + unsupported("buildPaths"); if (queryValidPaths(paths).size() != paths.size()) - unsupported(); + unsupported("buildPaths"); } @@ -586,15 +602,19 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; - // FIXME -#if 0 if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); auto info2 = make_ref<ValidPathInfo>(*info); info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; } -#endif if (info->ultimate) { auto info2 = make_ref<ValidPathInfo>(*info); @@ -609,6 +629,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, act.progress(total, info->narSize); }); srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); @@ -834,12 +856,11 @@ namespace nix { RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) { auto uri(uri_); - Store::Params params(extraParams); + Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { @@ -865,6 +886,15 @@ ref<Store> openStore(const std::string & uri_, } uri = uri_.substr(0, q); } + return {uri, params}; +} + +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); for (auto fun : *RegisterStoreImplementation::implementations) { auto store = fun(uri, params); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7c5b495a4..7fb568602 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -11,6 +11,8 @@ #include <atomic> #include <limits> #include <map> +#include <unordered_map> +#include <unordered_set> #include <memory> #include <string> @@ -23,6 +25,7 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */ MakeError(InvalidPath, Error) MakeError(Unsupported, Error) MakeError(SubstituteGone, Error) +MakeError(SubstituterDisabled, Error) struct BasicDerivation; @@ -46,7 +49,7 @@ const size_t storePathHashLen = 32; // i.e. 160 bits const uint32_t exportMagic = 0x4558494e; -typedef std::map<Path, Path> Roots; +typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; struct GCOptions @@ -348,7 +351,8 @@ public: (i.e. you'll get /nix/store/<hash> rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the full store path. */ - virtual PathSet queryAllValidPaths() = 0; + virtual PathSet queryAllValidPaths() + { unsupported("queryAllValidPaths"); } /* Query information about a valid path. It is permitted to omit the name part of the store path. */ @@ -356,19 +360,19 @@ public: /* Asynchronous version of queryPathInfo(). */ void queryPathInfo(const Path & path, - Callback<ref<ValidPathInfo>> callback); + Callback<ref<ValidPathInfo>> callback) noexcept; protected: virtual void queryPathInfoUncached(const Path & path, - Callback<std::shared_ptr<ValidPathInfo>> callback) = 0; + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0; public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ - virtual void queryReferrers(const Path & path, - PathSet & referrers) = 0; + virtual void queryReferrers(const Path & path, PathSet & referrers) + { unsupported("queryReferrers"); } /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the @@ -377,10 +381,12 @@ public: virtual PathSet queryValidDerivers(const Path & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) = 0; + virtual PathSet queryDerivationOutputs(const Path & path) + { unsupported("queryDerivationOutputs"); } /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) = 0; + virtual StringSet queryDerivationOutputNames(const Path & path) + { unsupported("queryDerivationOutputNames"); } /* Query the full store path given the hash part of a valid store path, or "" if the path doesn't exist. */ @@ -446,14 +452,16 @@ public: /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) = 0; + virtual void addTempRoot(const Path & path) + { unsupported("addTempRoot"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed to be a symlink to a store path. The garbage collector will automatically remove the indirect root when it finds that `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) = 0; + virtual void addIndirectRoot(const Path & path) + { unsupported("addIndirectRoot"); } /* Acquire the global GC lock, then immediately release it. This function must be called after registering a new permanent root, @@ -477,11 +485,15 @@ public: /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. */ - virtual Roots findRoots() = 0; + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) + { unsupported("findRoots"); } /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + virtual void collectGarbage(const GCOptions & options, GCResults & results) + { unsupported("collectGarbage"); } /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or @@ -512,11 +524,13 @@ public: virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; /* Return an object to access files in the Nix store. */ - virtual ref<FSAccessor> getFSAccessor() = 0; + virtual ref<FSAccessor> getFSAccessor() + { unsupported("getFSAccessor"); } /* Add signatures to the specified store path. The signatures are not verified. */ - virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; + virtual void addSignatures(const Path & storePath, const StringSet & sigs) + { unsupported("addSignatures"); } /* Utility functions. */ @@ -598,6 +612,12 @@ public: a notion of connection. Otherwise this is a no-op. */ virtual void connect() { }; + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() + { + return 0; + }; + /* Get the priority of the store, used to order substituters. In particular, binary caches can specify a priority field in their "nix-cache-info" file. Lower value means higher priority. */ @@ -613,9 +633,9 @@ protected: Stats stats; /* Unsupported methods. */ - [[noreturn]] void unsupported() + [[noreturn]] void unsupported(const std::string & op) { - throw Unsupported("requested operation is not supported by store '%s'", getUri()); + throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); } }; @@ -746,8 +766,7 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(), const std::string & stateDir = settings.nixStateDir); /* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options like - ‘binary-caches’. */ + ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); @@ -782,4 +801,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, for paths created by makeFixedOutputPath() / addToStore(). */ std::string makeFixedOutputCA(bool recursive, const Hash & hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); + } diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 1be8934a2..3aa120270 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -283,7 +283,7 @@ void parseDump(ParseSink & sink, Source & source) { string version; try { - version = readString(source); + version = readString(source, narVersionMagic1.size()); } catch (SerialisationError & e) { /* This generally means the integer at the start couldn't be decoded. Ignore and throw the exception below. */ @@ -331,7 +331,7 @@ struct RestoreSink : ParseSink filesystem doesn't support preallocation (e.g. on OpenSolaris). Since preallocation is just an optimisation, ignore it. */ - if (errno && errno != EINVAL) + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) throw SysError(format("preallocating file of %1% bytes") % len); } #endif diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 53b62f62a..0dd84e320 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -15,12 +15,10 @@ namespace nix { -static const size_t bufSize = 32 * 1024; - // Don't feed brotli too much at once. struct ChunkedCompressionSink : CompressionSink { - uint8_t outbuf[BUFSIZ]; + uint8_t outbuf[32 * 1024]; void write(const unsigned char * data, size_t len) override { @@ -124,7 +122,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink write(nullptr, 0); } - void writeInternal(const unsigned char * data, size_t len) + void writeInternal(const unsigned char * data, size_t len) override { assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); @@ -173,7 +171,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink writeInternal(nullptr, 0); } - void writeInternal(const unsigned char * data, size_t len) + void writeInternal(const unsigned char * data, size_t len) override { const uint8_t * next_in = data; size_t avail_in = len; @@ -252,7 +250,7 @@ struct XzCompressionSink : CompressionSink ret = lzma_stream_encoder_mt(&strm, &mt_options); done = true; #else - printMsg(lvlError, "warning: parallel compression requested but not supported for metho d '%1%', falling back to single-threaded compression", method); + printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); #endif } @@ -330,7 +328,7 @@ struct BzipCompressionSink : ChunkedCompressionSink writeInternal(nullptr, 0); } - void writeInternal(const unsigned char * data, size_t len) + void writeInternal(const unsigned char * data, size_t len) override { assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); @@ -380,7 +378,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink writeInternal(nullptr, 0); } - void writeInternal(const unsigned char * data, size_t len) + void writeInternal(const unsigned char * data, size_t len) override { const uint8_t * next_in = data; size_t avail_in = len; diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 9d82f13a5..1c14ebb18 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -105,9 +105,9 @@ string printHash16or32(const Hash & hash) std::string Hash::to_string(Base base, bool includeType) const { std::string s; - if (includeType) { + if (base == SRI || includeType) { s += printHashType(type); - s += ':'; + s += base == SRI ? '-' : ':'; } switch (base) { case Base16: @@ -117,6 +117,7 @@ std::string Hash::to_string(Base base, bool includeType) const s += printHash32(*this); break; case Base64: + case SRI: s += base64Encode(std::string((const char *) hash, hashSize)); break; } @@ -127,28 +128,33 @@ std::string Hash::to_string(Base base, bool includeType) const Hash::Hash(const std::string & s, HashType type) : type(type) { - auto colon = s.find(':'); - size_t pos = 0; - - if (colon == string::npos) { - if (type == htUnknown) + bool isSRI = false; + + auto sep = s.find(':'); + if (sep == string::npos) { + sep = s.find('-'); + if (sep != string::npos) { + isSRI = true; + } else if (type == htUnknown) throw BadHash("hash '%s' does not include a type", s); - } else { - string hts = string(s, 0, colon); + } + + if (sep != string::npos) { + string hts = string(s, 0, sep); this->type = parseHashType(hts); if (this->type == htUnknown) throw BadHash("unknown hash type '%s'", hts); if (type != htUnknown && type != this->type) throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); - pos = colon + 1; + pos = sep + 1; } init(); size_t size = s.size() - pos; - if (size == base16Len()) { + if (!isSRI && size == base16Len()) { auto parseHexDigit = [&](char c) { if (c >= '0' && c <= '9') return c - '0'; @@ -164,7 +170,7 @@ Hash::Hash(const std::string & s, HashType type) } } - else if (size == base32Len()) { + else if (!isSRI && size == base32Len()) { for (unsigned int n = 0; n < size; ++n) { char c = s[pos + size - n - 1]; @@ -187,10 +193,10 @@ Hash::Hash(const std::string & s, HashType type) } } - else if (size == base64Len()) { + else if (isSRI || size == base64Len()) { auto d = base64Decode(std::string(s, pos)); if (d.size() != hashSize) - throw BadHash("invalid base-64 hash '%s'", s); + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); assert(hashSize); memcpy(hash, d.data(), hashSize); } diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index fd7a61df8..2dbc3b630 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,7 +20,7 @@ const int sha512HashSize = 64; extern const string base32Chars; -enum Base : int { Base64, Base32, Base16 }; +enum Base : int { Base64, Base32, Base16, SRI }; struct Hash @@ -38,8 +38,9 @@ struct Hash Hash(HashType type) : type(type) { init(); }; /* Initialize the hash from a string representation, in the format - "[<type>:]<base16|base32|base64>". If the 'type' argument is - htUnknown, then the hash type must be specified in the + "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the string. */ Hash(const std::string & s, HashType type = htUnknown); diff --git a/src/libutil/json.hh b/src/libutil/json.hh index 02a39917f..45a22f011 100644 --- a/src/libutil/json.hh +++ b/src/libutil/json.hh @@ -170,7 +170,7 @@ public: ~JSONPlaceholder() { - assert(!first || std::uncaught_exception()); + assert(!first || std::uncaught_exceptions()); } template<typename T> diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 824f48fbf..e41a67d1f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,8 +6,4 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) -lboost_context - -libutil_LIBS = libformat - -libutil_CXXFLAGS = -DBROTLI=\"$(brotli)\" +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 799c6e1ae..b379306f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -21,7 +21,7 @@ Logger * logger = makeDefaultLogger(); void Logger::warn(const std::string & msg) { - log(lvlInfo, ANSI_RED "warning:" ANSI_NORMAL " " + msg); + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); } class SimpleLogger : public Logger @@ -46,6 +46,7 @@ public: char c; switch (lvl) { case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; default: c = '7'; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 678703102..5df03da74 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -6,6 +6,7 @@ namespace nix { typedef enum { lvlError = 0, + lvlWarn, lvlInfo, lvlTalkative, lvlChatty, @@ -25,6 +26,7 @@ typedef enum { actVerifyPaths = 107, actSubstitute = 108, actQueryPathInfo = 109, + actPostBuildHook = 110, } ActivityType; typedef enum { @@ -35,6 +37,7 @@ typedef enum { resSetPhase = 104, resProgress = 105, resSetExpected = 106, + resPostBuildLogLine = 107, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 9b8290e63..8b83f842c 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -2,7 +2,7 @@ #include <map> #include <list> -#include <experimental/optional> +#include <optional> namespace nix { @@ -64,7 +64,7 @@ public: /* Look up an item in the cache. If it exists, it becomes the most recently used item. */ - std::experimental::optional<Value> get(const Key & key) + std::optional<Value> get(const Key & key) { auto i = data.find(key); if (i == data.end()) return {}; diff --git a/src/libutil/pool.hh b/src/libutil/pool.hh index 0b142b059..d49067bb9 100644 --- a/src/libutil/pool.hh +++ b/src/libutil/pool.hh @@ -97,6 +97,7 @@ public: private: Pool & pool; std::shared_ptr<R> r; + bool bad = false; friend Pool; @@ -112,7 +113,8 @@ public: if (!r) return; { auto state_(pool.state.lock()); - state_->idle.push_back(ref<R>(r)); + if (!bad) + state_->idle.push_back(ref<R>(r)); assert(state_->inUse); state_->inUse--; } @@ -121,6 +123,8 @@ public: R * operator -> () { return &*r; } R & operator * () { return *r; } + + void markBad() { bad = true; } }; Handle get() diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index b2c49d911..8201549fd 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -161,21 +161,21 @@ size_t StringSource::read(unsigned char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif -std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) +std::unique_ptr<Source> sinkToSource( + std::function<void(Sink &)> fun, + std::function<void()> eof) { struct SinkToSource : Source { typedef boost::coroutines2::coroutine<std::string> coro_t; - coro_t::pull_type coro; + std::function<void(Sink &)> fun; + std::function<void()> eof; + std::optional<coro_t::pull_type> coro; + bool started = false; - SinkToSource(std::function<void(Sink &)> fun) - : coro([&](coro_t::push_type & yield) { - LambdaSink sink([&](const unsigned char * data, size_t len) { - if (len) yield(std::string((const char *) data, len)); - }); - fun(sink); - }) + SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) + : fun(fun), eof(eof) { } @@ -185,11 +185,18 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) size_t read(unsigned char * data, size_t len) override { if (!coro) - throw EndOfFile("coroutine has finished"); + coro = coro_t::pull_type([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }); + + if (!*coro) { eof(); abort(); } if (pos == cur.size()) { - if (!cur.empty()) coro(); - cur = coro.get(); + if (!cur.empty()) (*coro)(); + cur = coro->get(); pos = 0; } @@ -201,7 +208,7 @@ std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) } }; - return std::make_unique<SinkToSource>(fun); + return std::make_unique<SinkToSource>(fun, eof); } @@ -265,16 +272,17 @@ void readPadding(size_t len, Source & source) size_t readString(unsigned char * buf, size_t max, Source & source) { auto len = readNum<size_t>(source); - if (len > max) throw Error("string is too long"); + if (len > max) throw SerialisationError("string is too long"); source(buf, len); readPadding(len, source); return len; } -string readString(Source & source) +string readString(Source & source, size_t max) { auto len = readNum<size_t>(source); + if (len > max) throw SerialisationError("string is too long"); std::string res(len, 0); source((unsigned char*) res.data(), len); readPadding(len, source); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 14b62fdb6..a344a5ac7 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -179,6 +179,36 @@ struct TeeSource : Source } }; +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; /* Convert a function into a sink. */ struct LambdaSink : Sink @@ -214,7 +244,11 @@ struct LambdaSource : Source /* Convert a function that feeds data into a Sink into a Source. The Source executes the function as a coroutine. */ -std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun); +std::unique_ptr<Source> sinkToSource( + std::function<void(Sink &)> fun, + std::function<void()> eof = []() { + throw EndOfFile("coroutine has finished"); + }); void writePadding(size_t len, Sink & sink); @@ -280,7 +314,7 @@ inline uint64_t readLongLong(Source & source) void readPadding(size_t len, Source & source); size_t readString(unsigned char * buf, size_t max, Source & source); -string readString(Source & source); +string readString(Source & source, size_t max = std::numeric_limits<size_t>::max()); template<class T> T readStrings(Source & source); Source & operator >> (Source & in, string & s); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6bc64ae75..e65f8ee56 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -16,6 +16,7 @@ #include <future> #include <fcntl.h> +#include <grp.h> #include <limits.h> #include <pwd.h> #include <sys/ioctl.h> @@ -38,6 +39,9 @@ extern char * * environ; namespace nix { +const std::string nativeSystem = SYSTEM; + + BaseError & BaseError::addPrefix(const FormatOrString & fs) { prefix_ = fs.s + prefix_; @@ -80,6 +84,15 @@ void clearEnv() unsetenv(name.first.c_str()); } +void replaceEnv(std::map<std::string, std::string> newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + Path absPath(Path path, Path dir) { @@ -167,7 +180,7 @@ Path dirOf(const Path & path) { Path::size_type pos = path.rfind('/'); if (pos == string::npos) - throw Error(format("invalid file name '%1%'") % path); + return "."; return pos == 0 ? "/" : Path(path, 0, pos); } @@ -202,7 +215,7 @@ bool isInDir(const Path & path, const Path & dir) bool isDirOrInDir(const Path & path, const Path & dir) { - return path == dir or isInDir(path, dir); + return path == dir || isInDir(path, dir); } @@ -384,7 +397,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) } if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_blocks * 512; + bytesFreed += st.st_size; if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ @@ -468,7 +481,7 @@ static Lazy<Path> getHome2([]() { std::vector<char> buf(16384); struct passwd pwbuf; struct passwd * pw; - if (getpwuid_r(getuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 || !pw || !pw->pw_dir || !pw->pw_dir[0]) throw Error("cannot determine user's home directory"); homeDir = pw->pw_dir; @@ -496,6 +509,15 @@ Path getConfigDir() return configDir; } +std::vector<Path> getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + Path getDataDir() { @@ -956,7 +978,7 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) string runProgram(Path program, bool searchPath, const Strings & args, - const std::experimental::optional<std::string> & input) + const std::optional<std::string> & input) { RunOptions opts(program, args); opts.searchPath = searchPath; @@ -1006,13 +1028,35 @@ void runProgram2(const RunOptions & options) if (options.standardOut) out.create(); if (source) in.create(); + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + /* Fork. */ Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + Strings args_(options.args); args_.push_front(options.program); @@ -1024,7 +1068,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }); + }, processOptions); out.writeSide = -1; @@ -1125,7 +1169,7 @@ void _interrupted() /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ - if (!interruptThrown && !std::uncaught_exception()) { + if (!interruptThrown && !std::uncaught_exceptions()) { interruptThrown = true; throw Interrupted("interrupted by the user"); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index fc25d2775..07c3d28ff 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -14,7 +14,7 @@ #include <cstdio> #include <map> #include <sstream> -#include <experimental/optional> +#include <optional> #include <future> #ifndef HAVE_STRUCT_DIRENT_D_TYPE @@ -30,6 +30,10 @@ struct Sink; struct Source; +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + + /* Return an environment variable. */ string getEnv(const string & key, const string & def = ""); @@ -131,6 +135,9 @@ Path getCacheDir(); /* Return $XDG_CONFIG_HOME or $HOME/.config. */ Path getConfigDir(); +/* Return the directories to search for user configuration files */ +std::vector<Path> getConfigDirs(); + /* Return $XDG_DATA_HOME or $HOME/.local/share. */ Path getDataDir(); @@ -256,16 +263,21 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::experimental::optional<std::string> & input = {}); + const std::optional<std::string> & input = {}); struct RunOptions { + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; + std::optional<std::map<std::string, std::string>> environment; Path program; bool searchPath = true; Strings args; - std::experimental::optional<std::string> input; + std::optional<std::string> input; Source * standardIn = nullptr; Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; bool _killStderr = false; RunOptions(const Path & program, const Strings & args) @@ -398,6 +410,7 @@ void ignoreException(); /* Some ANSI escape sequences. */ #define ANSI_NORMAL "\e[0m" #define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" #define ANSI_RED "\e[31;1m" #define ANSI_GREEN "\e[32;1m" #define ANSI_BLUE "\e[34;1m" @@ -432,21 +445,34 @@ string get(const T & map, const string & key, const string & def = "") type T or an exception. (We abuse std::future<T> to pass the value or exception.) */ template<typename T> -struct Callback +class Callback { std::function<void(std::future<T>)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: Callback(std::function<void(std::future<T>)> fun) : fun(fun) { } - void operator()(T && t) const + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept { + auto prev = done.test_and_set(); + assert(!prev); std::promise<T> promise; promise.set_value(std::move(t)); fun(promise.get_future()); } - void rethrow(const std::exception_ptr & exc = std::current_exception()) const + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept { + auto prev = done.test_and_set(); + assert(!prev); std::promise<T> promise; promise.set_exception(exc); fun(promise.get_future()); diff --git a/src/linenoise/ConvertUTF.cpp b/src/linenoise/ConvertUTF.cpp deleted file mode 100644 index f7e5915d5..000000000 --- a/src/linenoise/ConvertUTF.cpp +++ /dev/null @@ -1,542 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Source code file. - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Sept 2001: fixed const & error conditions per - mods suggested by S. Parent & A. Lillich. - June 2002: Tim Dodd added detection and handling of incomplete - source sequences, enhanced error detection, added casts - to eliminate compiler warnings. - July 2003: slight mods to back out aggressive FFFE detection. - Jan 2004: updated switches in from-UTF8 conversions. - Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. - - See the header file "ConvertUTF.h" for complete documentation. - ------------------------------------------------------------------------- */ - -#include "ConvertUTF.h" -#ifdef CVTUTF_DEBUG -#include <stdio.h> -#endif - -namespace linenoise_ng { - -static const int halfShift = 10; /* used for shifting by 10 bits */ - -static const UTF32 halfBase = 0x0010000UL; -static const UTF32 halfMask = 0x3FFUL; - -#define UNI_SUR_HIGH_START (UTF32)0xD800 -#define UNI_SUR_HIGH_END (UTF32)0xDBFF -#define UNI_SUR_LOW_START (UTF32)0xDC00 -#define UNI_SUR_LOW_END (UTF32)0xDFFF -#define false 0 -#define true 1 - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - char16_t* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - if (target >= targetEnd) { - result = targetExhausted; break; - } - ch = *source++; - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_LEGAL_UTF32) { - if (flags == strictConversion) { - result = sourceIllegal; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - --source; /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF32* target = *targetStart; - UTF32 ch, ch2; - while (source < sourceEnd) { - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - if (target >= targetEnd) { - source = oldSource; /* Back up source pointer! */ - result = targetExhausted; break; - } - *target++ = ch; - } - *sourceStart = source; - *targetStart = target; -#ifdef CVTUTF_DEBUG -if (result == sourceIllegal) { - fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); - fflush(stderr); -} -#endif - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Index into the table below with the first byte of a UTF-8 sequence to - * get the number of trailing bytes that are supposed to follow it. - * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is - * left as-is for anyone who may want to do such conversion, which was - * allowed in earlier algorithms. - */ -static const char trailingBytesForUTF8[256] = { - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, - 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, - 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 -}; - -/* - * Magic values subtracted from a buffer value during UTF8 conversion. - * This table contains as many values as there might be trailing bytes - * in a UTF-8 sequence. - */ -static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, - 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; - -/* - * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed - * into the first byte, depending on how many bytes follow. There are - * as many entries in this table as there are UTF-8 sequence types. - * (I.e., one byte sequence, two byte... etc.). Remember that sequencs - * for *legal* UTF-8 will be 4 or fewer bytes total. - */ -static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; - -/* --------------------------------------------------------------------- */ - -/* The interface converts a whole buffer to avoid function-call overhead. - * Constants have been gathered. Loops & conditionals have been removed as - * much as possible for efficiency, in favor of drop-through switches. - * (See "Note A" at the bottom of the file for equivalent code.) - * If your compiler supports it, the "isLegalUTF8" call can be turned - * into an inline function. - */ - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF16* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ - ch = *source++; - /* If we have a surrogate pair, convert to UTF32 first. */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { - /* If the 16 bits following the high surrogate are in the source buffer... */ - if (source < sourceEnd) { - UTF32 ch2 = *source; - /* If it's a low surrogate, convert to UTF32. */ - if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { - ch = ((ch - UNI_SUR_HIGH_START) << halfShift) - + (ch2 - UNI_SUR_LOW_START) + halfBase; - ++source; - } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } else { /* We don't have the 16 bits following the high surrogate. */ - --source; /* return to the high surrogate */ - result = sourceExhausted; - break; - } - } else if (flags == strictConversion) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* Figure out how many bytes the result will require */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - } - - target += bytesToWrite; - if (target > targetEnd) { - source = oldSource; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -/* - * Utility routine to tell whether a sequence of bytes is legal UTF-8. - * This must be called with the length pre-determined by the first byte. - * If not calling this from ConvertUTF8to*, then the length can be set by: - * length = trailingBytesForUTF8[*source]+1; - * and the sequence is illegal right away if there aren't that many bytes - * available. - * If presented with a length > 4, this returns false. The Unicode - * definition of UTF-8 goes up to 4-byte sequences. - */ - -static Boolean isLegalUTF8(const UTF8 *source, int length) { - UTF8 a; - const UTF8 *srcptr = source+length; - switch (length) { - default: return false; - /* Everything else falls through when "true"... */ - case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; - case 2: if ((a = (*--srcptr)) > 0xBF) return false; - - switch (*source) { - /* no fall-through in this inner switch */ - case 0xE0: if (a < 0xA0) return false; break; - case 0xED: if (a > 0x9F) return false; break; - case 0xF0: if (a < 0x90) return false; break; - case 0xF4: if (a > 0x8F) return false; break; - default: if (a < 0x80) return false; - } - - case 1: if (*source >= 0x80 && *source < 0xC2) return false; - } - if (*source > 0xF4) return false; - return true; -} - -/* --------------------------------------------------------------------- */ - -/* - * Exported function to return whether a UTF-8 sequence is legal or not. - * This is not used here; it's just exported. - */ -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { - int length = trailingBytesForUTF8[*source]+1; - if (source+length > sourceEnd) { - return false; - } - return isLegalUTF8(source, length); -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF16* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = (UTF16)ch; /* normal case */ - } - } else if (ch > UNI_MAX_UTF16) { - if (flags == strictConversion) { - result = sourceIllegal; - source -= (extraBytesToRead+1); /* return to the start */ - break; /* Bail out; shouldn't continue */ - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - /* target is a character in range 0xFFFF - 0x10FFFF. */ - if (target + 1 >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up source pointer! */ - result = targetExhausted; break; - } - ch -= halfBase; - *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); - *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF32* source = *sourceStart; - UTF8* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch; - unsigned short bytesToWrite = 0; - const UTF32 byteMask = 0xBF; - const UTF32 byteMark = 0x80; - ch = *source++; - if (flags == strictConversion ) { - /* UTF-16 surrogate values are illegal in UTF-32 */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - --source; /* return to the illegal value itself */ - result = sourceIllegal; - break; - } - } - /* - * Figure out how many bytes the result will require. Turn any - * illegally large UTF32 things (> Plane 17) into replacement chars. - */ - if (ch < (UTF32)0x80) { bytesToWrite = 1; - } else if (ch < (UTF32)0x800) { bytesToWrite = 2; - } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; - } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; - } else { bytesToWrite = 3; - ch = UNI_REPLACEMENT_CHAR; - result = sourceIllegal; - } - - target += bytesToWrite; - if (target > targetEnd) { - --source; /* Back up source pointer! */ - target -= bytesToWrite; result = targetExhausted; break; - } - switch (bytesToWrite) { /* note: everything falls through. */ - case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; - case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); - } - target += bytesToWrite; - } - *sourceStart = source; - *targetStart = target; - return result; -} - -/* --------------------------------------------------------------------- */ - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { - ConversionResult result = conversionOK; - const UTF8* source = *sourceStart; - UTF32* target = *targetStart; - while (source < sourceEnd) { - UTF32 ch = 0; - unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; - if (source + extraBytesToRead >= sourceEnd) { - result = sourceExhausted; break; - } - /* Do this check whether lenient or strict */ - if (! isLegalUTF8(source, extraBytesToRead+1)) { - result = sourceIllegal; - break; - } - /* - * The cases all fall through. See "Note A" below. - */ - switch (extraBytesToRead) { - case 5: ch += *source++; ch <<= 6; - case 4: ch += *source++; ch <<= 6; - case 3: ch += *source++; ch <<= 6; - case 2: ch += *source++; ch <<= 6; - case 1: ch += *source++; ch <<= 6; - case 0: ch += *source++; - } - ch -= offsetsFromUTF8[extraBytesToRead]; - - if (target >= targetEnd) { - source -= (extraBytesToRead+1); /* Back up the source pointer! */ - result = targetExhausted; break; - } - if (ch <= UNI_MAX_LEGAL_UTF32) { - /* - * UTF-16 surrogate values are illegal in UTF-32, and anything - * over Plane 17 (> 0x10FFFF) is illegal. - */ - if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { - if (flags == strictConversion) { - source -= (extraBytesToRead+1); /* return to the illegal value itself */ - result = sourceIllegal; - break; - } else { - *target++ = UNI_REPLACEMENT_CHAR; - } - } else { - *target++ = ch; - } - } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ - result = sourceIllegal; - *target++ = UNI_REPLACEMENT_CHAR; - } - } - *sourceStart = source; - *targetStart = target; - return result; -} - -} - -/* --------------------------------------------------------------------- - - Note A. - The fall-through switches in UTF-8 reading code save a - temp variable, some decrements & conditionals. The switches - are equivalent to the following loop: - { - int tmpBytesToRead = extraBytesToRead+1; - do { - ch += *source++; - --tmpBytesToRead; - if (tmpBytesToRead) ch <<= 6; - } while (tmpBytesToRead > 0); - } - In UTF-8 writing code, the switches on "bytesToWrite" are - similarly unrolled loops. - - --------------------------------------------------------------------- */ diff --git a/src/linenoise/ConvertUTF.h b/src/linenoise/ConvertUTF.h deleted file mode 100755 index 8a296235d..000000000 --- a/src/linenoise/ConvertUTF.h +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2001-2004 Unicode, Inc. - * - * Disclaimer - * - * This source code is provided as is by Unicode, Inc. No claims are - * made as to fitness for any particular purpose. No warranties of any - * kind are expressed or implied. The recipient agrees to determine - * applicability of information provided. If this file has been - * purchased on magnetic or optical media from Unicode, Inc., the - * sole remedy for any claim will be exchange of defective media - * within 90 days of receipt. - * - * Limitations on Rights to Redistribute This Code - * - * Unicode, Inc. hereby grants the right to freely use the information - * supplied in this file in the creation of products supporting the - * Unicode Standard, and to make copies of this file in any form - * for internal or external distribution as long as this notice - * remains attached. - */ - -/* --------------------------------------------------------------------- - - Conversions between UTF32, UTF-16, and UTF-8. Header file. - - Several funtions are included here, forming a complete set of - conversions between the three formats. UTF-7 is not included - here, but is handled in a separate source file. - - Each of these routines takes pointers to input buffers and output - buffers. The input buffers are const. - - Each routine converts the text between *sourceStart and sourceEnd, - putting the result into the buffer between *targetStart and - targetEnd. Note: the end pointers are *after* the last item: e.g. - *(sourceEnd - 1) is the last item. - - The return result indicates whether the conversion was successful, - and if not, whether the problem was in the source or target buffers. - (Only the first encountered problem is indicated.) - - After the conversion, *sourceStart and *targetStart are both - updated to point to the end of last text successfully converted in - the respective buffers. - - Input parameters: - sourceStart - pointer to a pointer to the source buffer. - The contents of this are modified on return so that - it points at the next thing to be converted. - targetStart - similarly, pointer to pointer to the target buffer. - sourceEnd, targetEnd - respectively pointers to the ends of the - two buffers, for overflow checking only. - - These conversion functions take a ConversionFlags argument. When this - flag is set to strict, both irregular sequences and isolated surrogates - will cause an error. When the flag is set to lenient, both irregular - sequences and isolated surrogates are converted. - - Whether the flag is strict or lenient, all illegal sequences will cause - an error return. This includes sequences such as: <F4 90 80 80>, <C0 80>, - or <A0> in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code - must check for illegal sequences. - - When the flag is set to lenient, characters over 0x10FFFF are converted - to the replacement character; otherwise (when the flag is set to strict) - they constitute an error. - - Output parameters: - The value "sourceIllegal" is returned from some routines if the input - sequence is malformed. When "sourceIllegal" is returned, the source - value will point to the illegal value that caused the problem. E.g., - in UTF-8 when a sequence is malformed, it points to the start of the - malformed sequence. - - Author: Mark E. Davis, 1994. - Rev History: Rick McGowan, fixes & updates May 2001. - Fixes & updates, Sept 2001. - ------------------------------------------------------------------------- */ - -/* --------------------------------------------------------------------- - The following 4 definitions are compiler-specific. - The C standard does not guarantee that wchar_t has at least - 16 bits, so wchar_t is no less portable than unsigned short! - All should be unsigned values to avoid sign extension during - bit mask & shift operations. ------------------------------------------------------------------------- */ - -#if 0 -typedef unsigned long UTF32; /* at least 32 bits */ -typedef unsigned short UTF16; /* at least 16 bits */ -typedef unsigned char UTF8; /* typically 8 bits */ -#endif - -#include <stdint.h> -#include <string> - -namespace linenoise_ng { - -typedef uint32_t UTF32; -typedef uint16_t UTF16; -typedef uint8_t UTF8; -typedef unsigned char Boolean; /* 0 or 1 */ - -/* Some fundamental constants */ -#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD -#define UNI_MAX_BMP (UTF32)0x0000FFFF -#define UNI_MAX_UTF16 (UTF32)0x0010FFFF -#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF -#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF - -typedef enum { - conversionOK, /* conversion successful */ - sourceExhausted, /* partial character in source, but hit end */ - targetExhausted, /* insuff. room in target for conversion */ - sourceIllegal /* source sequence is illegal/malformed */ -} ConversionResult; - -typedef enum { - strictConversion = 0, - lenientConversion -} ConversionFlags; - -// /* This is for C++ and does no harm in C */ -// #ifdef __cplusplus -// extern "C" { -// #endif - -ConversionResult ConvertUTF8toUTF16 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF8 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF8toUTF32 ( - const UTF8** sourceStart, const UTF8* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF8 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF16toUTF32 ( - const UTF16** sourceStart, const UTF16* sourceEnd, - UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); - -ConversionResult ConvertUTF32toUTF16 ( - const UTF32** sourceStart, const UTF32* sourceEnd, - char16_t** targetStart, char16_t* targetEnd, ConversionFlags flags); - -Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); - -// #ifdef __cplusplus -// } -// #endif - -} - -/* --------------------------------------------------------------------- */ diff --git a/src/linenoise/LICENSE b/src/linenoise/LICENSE deleted file mode 100644 index b7c58c445..000000000 --- a/src/linenoise/LICENSE +++ /dev/null @@ -1,66 +0,0 @@ -linenoise.cpp -============= - -Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> -Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of Redis nor the names of its contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - - -wcwidth.cpp -=========== - -Markus Kuhn -- 2007-05-26 (Unicode 5.0) - -Permission to use, copy, modify, and distribute this software -for any purpose and without fee is hereby granted. The author -disclaims all warranties with regard to this software. - - - -ConvertUTF.cpp -============== - -Copyright 2001-2004 Unicode, Inc. - -Disclaimer - -This source code is provided as is by Unicode, Inc. No claims are -made as to fitness for any particular purpose. No warranties of any -kind are expressed or implied. The recipient agrees to determine -applicability of information provided. If this file has been -purchased on magnetic or optical media from Unicode, Inc., the -sole remedy for any claim will be exchange of defective media -within 90 days of receipt. - -Limitations on Rights to Redistribute This Code - -Unicode, Inc. hereby grants the right to freely use the information -supplied in this file in the creation of products supporting the -Unicode Standard, and to make copies of this file in any form -for internal or external distribution as long as this notice -remains attached. diff --git a/src/linenoise/linenoise.cpp b/src/linenoise/linenoise.cpp deleted file mode 100644 index c57505d2f..000000000 --- a/src/linenoise/linenoise.cpp +++ /dev/null @@ -1,3450 +0,0 @@ -/* linenoise.c -- guerrilla line editing library against the idea that a - * line editing lib needs to be 20,000 lines of C code. - * - * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> - * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * line editing lib needs to be 20,000 lines of C code. - * - * You can find the latest source code at: - * - * http://github.com/antirez/linenoise - * - * Does a number of crazy assumptions that happen to be true in 99.9999% of - * the 2010 UNIX computers around. - * - * References: - * - http://invisible-island.net/xterm/ctlseqs/ctlseqs.html - * - http://www.3waylabs.com/nw/WWW/products/wizcon/vt220.html - * - * Todo list: - * - Switch to gets() if $TERM is something we can't support. - * - Filter bogus Ctrl+<char> combinations. - * - Win32 support - * - * Bloat: - * - Completion? - * - History search like Ctrl+r in readline? - * - * List of escape sequences used by this program, we do everything just - * with three sequences. In order to be so cheap we may have some - * flickering effect with some slow terminal, but the lesser sequences - * the more compatible. - * - * CHA (Cursor Horizontal Absolute) - * Sequence: ESC [ n G - * Effect: moves cursor to column n (1 based) - * - * EL (Erase Line) - * Sequence: ESC [ n K - * Effect: if n is 0 or missing, clear from cursor to end of line - * Effect: if n is 1, clear from beginning of line to cursor - * Effect: if n is 2, clear entire line - * - * CUF (Cursor Forward) - * Sequence: ESC [ n C - * Effect: moves cursor forward of n chars - * - * The following are used to clear the screen: ESC [ H ESC [ 2 J - * This is actually composed of two sequences: - * - * cursorhome - * Sequence: ESC [ H - * Effect: moves the cursor to upper left corner - * - * ED2 (Clear entire screen) - * Sequence: ESC [ 2 J - * Effect: clear the whole screen - * - */ - -#ifdef _WIN32 - -#include <conio.h> -#include <windows.h> -#include <io.h> - -#if defined(_MSC_VER) && _MSC_VER < 1900 -#define snprintf _snprintf // Microsoft headers use underscores in some names -#endif - -#if !defined GNUC -#define strcasecmp _stricmp -#endif - -#define strdup _strdup -#define isatty _isatty -#define write _write -#define STDIN_FILENO 0 - -#else /* _WIN32 */ - -#include <signal.h> -#include <termios.h> -#include <unistd.h> -#include <stdlib.h> -#include <string.h> -#include <sys/types.h> -#include <sys/ioctl.h> -#include <cctype> -#include <wctype.h> - -#endif /* _WIN32 */ - -#include <stdio.h> -#include <errno.h> -#include <fcntl.h> - -#include "linenoise.h" -#include "ConvertUTF.h" - -#include <string> -#include <vector> -#include <memory> - -using std::string; -using std::vector; -using std::unique_ptr; -using namespace linenoise_ng; - -typedef unsigned char char8_t; - -static ConversionResult copyString8to32(char32_t* dst, size_t dstSize, - size_t& dstCount, const char* src) { - const UTF8* sourceStart = reinterpret_cast<const UTF8*>(src); - const UTF8* sourceEnd = sourceStart + strlen(src); - UTF32* targetStart = reinterpret_cast<UTF32*>(dst); - UTF32* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF8toUTF32( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - dstCount = targetStart - reinterpret_cast<UTF32*>(dst); - - if (dstCount < dstSize) { - *targetStart = 0; - } - } - - return res; -} - -static ConversionResult copyString8to32(char32_t* dst, size_t dstSize, - size_t& dstCount, const char8_t* src) { - return copyString8to32(dst, dstSize, dstCount, - reinterpret_cast<const char*>(src)); -} - -static size_t strlen32(const char32_t* str) { - const char32_t* ptr = str; - - while (*ptr) { - ++ptr; - } - - return ptr - str; -} - -static size_t strlen8(const char8_t* str) { - return strlen(reinterpret_cast<const char*>(str)); -} - -static char8_t* strdup8(const char* src) { - return reinterpret_cast<char8_t*>(strdup(src)); -} - -#ifdef _WIN32 -static const int FOREGROUND_WHITE = - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; -static const int BACKGROUND_WHITE = - BACKGROUND_RED | BACKGROUND_GREEN | BACKGROUND_BLUE; -static const int INTENSITY = FOREGROUND_INTENSITY | BACKGROUND_INTENSITY; - -class WinAttributes { - public: - WinAttributes() { - CONSOLE_SCREEN_BUFFER_INFO info; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info); - _defaultAttribute = info.wAttributes & INTENSITY; - _defaultColor = info.wAttributes & FOREGROUND_WHITE; - _defaultBackground = info.wAttributes & BACKGROUND_WHITE; - - _consoleAttribute = _defaultAttribute; - _consoleColor = _defaultColor | _defaultBackground; - } - - public: - int _defaultAttribute; - int _defaultColor; - int _defaultBackground; - - int _consoleAttribute; - int _consoleColor; -}; - -static WinAttributes WIN_ATTR; - -static void copyString32to16(char16_t* dst, size_t dstSize, size_t* dstCount, - const char32_t* src, size_t srcSize) { - const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src); - const UTF32* sourceEnd = sourceStart + srcSize; - char16_t* targetStart = reinterpret_cast<char16_t*>(dst); - char16_t* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF32toUTF16( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - *dstCount = targetStart - reinterpret_cast<char16_t*>(dst); - - if (*dstCount < dstSize) { - *targetStart = 0; - } - } -} -#endif - -static void copyString32to8(char* dst, size_t dstSize, size_t* dstCount, - const char32_t* src, size_t srcSize) { - const UTF32* sourceStart = reinterpret_cast<const UTF32*>(src); - const UTF32* sourceEnd = sourceStart + srcSize; - UTF8* targetStart = reinterpret_cast<UTF8*>(dst); - UTF8* targetEnd = targetStart + dstSize; - - ConversionResult res = ConvertUTF32toUTF8( - &sourceStart, sourceEnd, &targetStart, targetEnd, lenientConversion); - - if (res == conversionOK) { - *dstCount = targetStart - reinterpret_cast<UTF8*>(dst); - - if (*dstCount < dstSize) { - *targetStart = 0; - } - } -} - -static void copyString32to8(char* dst, size_t dstLen, const char32_t* src) { - size_t dstCount = 0; - copyString32to8(dst, dstLen, &dstCount, src, strlen32(src)); -} - -static void copyString32(char32_t* dst, const char32_t* src, size_t len) { - while (0 < len && *src) { - *dst++ = *src++; - --len; - } - - *dst = 0; -} - -static int strncmp32(const char32_t* left, const char32_t* right, size_t len) { - while (0 < len && *left) { - if (*left != *right) { - return *left - *right; - } - - ++left; - ++right; - --len; - } - - return 0; -} - -#ifdef _WIN32 -#include <iostream> - -static size_t OutputWin(char16_t* text16, char32_t* text32, size_t len32) { - size_t count16 = 0; - - copyString32to16(text16, len32, &count16, text32, len32); - WriteConsoleW(GetStdHandle(STD_OUTPUT_HANDLE), text16, - static_cast<DWORD>(count16), nullptr, nullptr); - - return count16; -} - -static char32_t* HandleEsc(char32_t* p, char32_t* end) { - if (*p == '[') { - int code = 0; - - for (++p; p < end; ++p) { - char32_t c = *p; - - if ('0' <= c && c <= '9') { - code = code * 10 + (c - '0'); - } else if (c == 'm' || c == ';') { - switch (code) { - case 0: - WIN_ATTR._consoleAttribute = WIN_ATTR._defaultAttribute; - WIN_ATTR._consoleColor = - WIN_ATTR._defaultColor | WIN_ATTR._defaultBackground; - break; - - case 1: // BOLD - case 5: // BLINK - WIN_ATTR._consoleAttribute = - (WIN_ATTR._defaultAttribute ^ FOREGROUND_INTENSITY) & INTENSITY; - break; - - case 30: - WIN_ATTR._consoleColor = BACKGROUND_WHITE; - break; - - case 31: - WIN_ATTR._consoleColor = - FOREGROUND_RED | WIN_ATTR._defaultBackground; - break; - - case 32: - WIN_ATTR._consoleColor = - FOREGROUND_GREEN | WIN_ATTR._defaultBackground; - break; - - case 33: - WIN_ATTR._consoleColor = - FOREGROUND_RED | FOREGROUND_GREEN | WIN_ATTR._defaultBackground; - break; - - case 34: - WIN_ATTR._consoleColor = - FOREGROUND_BLUE | WIN_ATTR._defaultBackground; - break; - - case 35: - WIN_ATTR._consoleColor = - FOREGROUND_BLUE | FOREGROUND_RED | WIN_ATTR._defaultBackground; - break; - - case 36: - WIN_ATTR._consoleColor = FOREGROUND_BLUE | FOREGROUND_GREEN | - WIN_ATTR._defaultBackground; - break; - - case 37: - WIN_ATTR._consoleColor = FOREGROUND_GREEN | FOREGROUND_RED | - FOREGROUND_BLUE | - WIN_ATTR._defaultBackground; - break; - } - - code = 0; - } - - if (*p == 'm') { - ++p; - break; - } - } - } else { - ++p; - } - - auto handle = GetStdHandle(STD_OUTPUT_HANDLE); - SetConsoleTextAttribute(handle, - WIN_ATTR._consoleAttribute | WIN_ATTR._consoleColor); - - return p; -} - -static size_t WinWrite32(char16_t* text16, char32_t* text32, size_t len32) { - char32_t* p = text32; - char32_t* q = p; - char32_t* e = text32 + len32; - size_t count16 = 0; - - while (p < e) { - if (*p == 27) { - if (q < p) { - count16 += OutputWin(text16, q, p - q); - } - - q = p = HandleEsc(p + 1, e); - } else { - ++p; - } - } - - if (q < p) { - count16 += OutputWin(text16, q, p - q); - } - - return count16; -} -#endif - -static int write32(int fd, char32_t* text32, int len32) { -#ifdef _WIN32 - if (isatty(fd)) { - size_t len16 = 2 * len32 + 1; - unique_ptr<char16_t[]> text16(new char16_t[len16]); - size_t count16 = WinWrite32(text16.get(), text32, len32); - - return static_cast<int>(count16); - } else { - size_t len8 = 4 * len32 + 1; - unique_ptr<char[]> text8(new char[len8]); - size_t count8 = 0; - - copyString32to8(text8.get(), len8, &count8, text32, len32); - - return write(fd, text8.get(), static_cast<unsigned int>(count8)); - } -#else - size_t len8 = 4 * len32 + 1; - unique_ptr<char[]> text8(new char[len8]); - size_t count8 = 0; - - copyString32to8(text8.get(), len8, &count8, text32, len32); - - return write(fd, text8.get(), count8); -#endif -} - -class Utf32String { - public: - Utf32String() : _length(0), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[1](); - } - - explicit Utf32String(const char* src) : _length(0), _data(nullptr) { - size_t len = strlen(src); - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - copyString8to32(_data, len + 1, _length, src); - } - - explicit Utf32String(const char8_t* src) : _length(0), _data(nullptr) { - size_t len = strlen(reinterpret_cast<const char*>(src)); - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - copyString8to32(_data, len + 1, _length, src); - } - - explicit Utf32String(const char32_t* src) : _length(0), _data(nullptr) { - for (_length = 0; src[_length] != 0; ++_length) { - } - - // note: parens intentional, _data must be properly initialized - _data = new char32_t[_length + 1](); - memcpy(_data, src, _length * sizeof(char32_t)); - } - - explicit Utf32String(const char32_t* src, int len) : _length(len), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len + 1](); - memcpy(_data, src, len * sizeof(char32_t)); - } - - explicit Utf32String(int len) : _length(0), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[len](); - } - - explicit Utf32String(const Utf32String& that) : _length(that._length), _data(nullptr) { - // note: parens intentional, _data must be properly initialized - _data = new char32_t[_length + 1](); - memcpy(_data, that._data, sizeof(char32_t) * _length); - } - - Utf32String& operator=(const Utf32String& that) { - if (this != &that) { - delete[] _data; - _data = new char32_t[that._length](); - _length = that._length; - memcpy(_data, that._data, sizeof(char32_t) * _length); - } - - return *this; - } - - ~Utf32String() { delete[] _data; } - - public: - char32_t* get() const { return _data; } - - size_t length() const { return _length; } - - size_t chars() const { return _length; } - - void initFromBuffer() { - for (_length = 0; _data[_length] != 0; ++_length) { - } - } - - const char32_t& operator[](size_t pos) const { return _data[pos]; } - - char32_t& operator[](size_t pos) { return _data[pos]; } - - private: - size_t _length; - char32_t* _data; -}; - -class Utf8String { - Utf8String(const Utf8String&) = delete; - Utf8String& operator=(const Utf8String&) = delete; - - public: - explicit Utf8String(const Utf32String& src) { - size_t len = src.length() * 4 + 1; - _data = new char[len]; - copyString32to8(_data, len, src.get()); - } - - ~Utf8String() { delete[] _data; } - - public: - char* get() const { return _data; } - - private: - char* _data; -}; - -struct linenoiseCompletions { - vector<Utf32String> completionStrings; -}; - -#define LINENOISE_DEFAULT_HISTORY_MAX_LEN 100 -#define LINENOISE_MAX_LINE 4096 - -// make control-characters more readable -#define ctrlChar(upperCaseASCII) (upperCaseASCII - 0x40) - -/** - * Recompute widths of all characters in a char32_t buffer - * @param text input buffer of Unicode characters - * @param widths output buffer of character widths - * @param charCount number of characters in buffer - */ -namespace linenoise_ng { -int mk_wcwidth(char32_t ucs); -} - -static void recomputeCharacterWidths(const char32_t* text, char* widths, - int charCount) { - for (int i = 0; i < charCount; ++i) { - widths[i] = mk_wcwidth(text[i]); - } -} - -/** - * Calculate a new screen position given a starting position, screen width and - * character count - * @param x initial x position (zero-based) - * @param y initial y position (zero-based) - * @param screenColumns screen column count - * @param charCount character positions to advance - * @param xOut returned x position (zero-based) - * @param yOut returned y position (zero-based) - */ -static void calculateScreenPosition(int x, int y, int screenColumns, - int charCount, int& xOut, int& yOut) { - xOut = x; - yOut = y; - int charsRemaining = charCount; - while (charsRemaining > 0) { - int charsThisRow = (x + charsRemaining < screenColumns) ? charsRemaining - : screenColumns - x; - xOut = x + charsThisRow; - yOut = y; - charsRemaining -= charsThisRow; - x = 0; - ++y; - } - if (xOut == screenColumns) { // we have to special-case line wrap - xOut = 0; - ++yOut; - } -} - -/** - * Calculate a column width using mk_wcswidth() - * @param buf32 text to calculate - * @param len length of text to calculate - */ -namespace linenoise_ng { -int mk_wcswidth(const char32_t* pwcs, size_t n); -} - -static int calculateColumnPosition(char32_t* buf32, int len) { - int width = mk_wcswidth(reinterpret_cast<const char32_t*>(buf32), len); - if (width == -1) - return len; - else - return width; -} - -static bool isControlChar(char32_t testChar) { - return (testChar < ' ') || // C0 controls - (testChar >= 0x7F && testChar <= 0x9F); // DEL and C1 controls -} - -struct PromptBase { // a convenience struct for grouping prompt info - Utf32String promptText; // our copy of the prompt text, edited - char* promptCharWidths; // character widths from mk_wcwidth() - int promptChars; // chars in promptText - int promptBytes; // bytes in promptText - int promptExtraLines; // extra lines (beyond 1) occupied by prompt - int promptIndentation; // column offset to end of prompt - int promptLastLinePosition; // index into promptText where last line begins - int promptPreviousInputLen; // promptChars of previous input line, for - // clearing - int promptCursorRowOffset; // where the cursor is relative to the start of - // the prompt - int promptScreenColumns; // width of screen in columns - int promptPreviousLen; // help erasing - int promptErrorCode; // error code (invalid UTF-8) or zero - - PromptBase() : promptPreviousInputLen(0) {} - - bool write() { - if (write32(1, promptText.get(), promptBytes) == -1) return false; - - return true; - } -}; - -struct PromptInfo : public PromptBase { - PromptInfo(const char* textPtr, int columns) { - promptExtraLines = 0; - promptLastLinePosition = 0; - promptPreviousLen = 0; - promptScreenColumns = columns; - Utf32String tempUnicode(textPtr); - - // strip control characters from the prompt -- we do allow newline - char32_t* pIn = tempUnicode.get(); - char32_t* pOut = pIn; - - int len = 0; - int x = 0; - - bool const strip = (isatty(1) == 0); - - while (*pIn) { - char32_t c = *pIn; - if ('\n' == c || !isControlChar(c)) { - *pOut = c; - ++pOut; - ++pIn; - ++len; - if ('\n' == c || ++x >= promptScreenColumns) { - x = 0; - ++promptExtraLines; - promptLastLinePosition = len; - } - } else if (c == '\x1b') { - if (strip) { - // jump over control chars - ++pIn; - if (*pIn == '[') { - ++pIn; - while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) { - ++pIn; - } - if (*pIn == 'm') { - ++pIn; - } - } - } else { - // copy control chars - *pOut = *pIn; - ++pOut; - ++pIn; - if (*pIn == '[') { - *pOut = *pIn; - ++pOut; - ++pIn; - while (*pIn && ((*pIn == ';') || ((*pIn >= '0' && *pIn <= '9')))) { - *pOut = *pIn; - ++pOut; - ++pIn; - } - if (*pIn == 'm') { - *pOut = *pIn; - ++pOut; - ++pIn; - } - } - } - } else { - ++pIn; - } - } - *pOut = 0; - promptChars = len; - promptBytes = static_cast<int>(pOut - tempUnicode.get()); - promptText = tempUnicode; - - promptIndentation = len - promptLastLinePosition; - promptCursorRowOffset = promptExtraLines; - } -}; - -// Used with DynamicPrompt (history search) -// -static const Utf32String forwardSearchBasePrompt("(i-search)`"); -static const Utf32String reverseSearchBasePrompt("(reverse-i-search)`"); -static const Utf32String endSearchBasePrompt("': "); -static Utf32String - previousSearchText; // remembered across invocations of linenoise() - -// changing prompt for "(reverse-i-search)`text':" etc. -// -struct DynamicPrompt : public PromptBase { - Utf32String searchText; // text we are searching for - char* searchCharWidths; // character widths from mk_wcwidth() - int searchTextLen; // chars in searchText - int direction; // current search direction, 1=forward, -1=reverse - - DynamicPrompt(PromptBase& pi, int initialDirection) - : searchTextLen(0), direction(initialDirection) { - promptScreenColumns = pi.promptScreenColumns; - promptCursorRowOffset = 0; - Utf32String emptyString(1); - searchText = emptyString; - const Utf32String* basePrompt = - (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt; - size_t promptStartLength = basePrompt->length(); - promptChars = - static_cast<int>(promptStartLength + endSearchBasePrompt.length()); - promptBytes = promptChars; - promptLastLinePosition = promptChars; // TODO fix this, we are asssuming - // that the history prompt won't wrap - // (!) - promptPreviousLen = promptChars; - Utf32String tempUnicode(promptChars + 1); - memcpy(tempUnicode.get(), basePrompt->get(), - sizeof(char32_t) * promptStartLength); - memcpy(&tempUnicode[promptStartLength], endSearchBasePrompt.get(), - sizeof(char32_t) * (endSearchBasePrompt.length() + 1)); - tempUnicode.initFromBuffer(); - promptText = tempUnicode; - calculateScreenPosition(0, 0, pi.promptScreenColumns, promptChars, - promptIndentation, promptExtraLines); - } - - void updateSearchPrompt(void) { - const Utf32String* basePrompt = - (direction > 0) ? &forwardSearchBasePrompt : &reverseSearchBasePrompt; - size_t promptStartLength = basePrompt->length(); - promptChars = static_cast<int>(promptStartLength + searchTextLen + - endSearchBasePrompt.length()); - promptBytes = promptChars; - Utf32String tempUnicode(promptChars + 1); - memcpy(tempUnicode.get(), basePrompt->get(), - sizeof(char32_t) * promptStartLength); - memcpy(&tempUnicode[promptStartLength], searchText.get(), - sizeof(char32_t) * searchTextLen); - size_t endIndex = promptStartLength + searchTextLen; - memcpy(&tempUnicode[endIndex], endSearchBasePrompt.get(), - sizeof(char32_t) * (endSearchBasePrompt.length() + 1)); - tempUnicode.initFromBuffer(); - promptText = tempUnicode; - } - - void updateSearchText(const char32_t* textPtr) { - Utf32String tempUnicode(textPtr); - searchTextLen = static_cast<int>(tempUnicode.chars()); - searchText = tempUnicode; - updateSearchPrompt(); - } -}; - -class KillRing { - static const int capacity = 10; - int size; - int index; - char indexToSlot[10]; - vector<Utf32String> theRing; - - public: - enum action { actionOther, actionKill, actionYank }; - action lastAction; - size_t lastYankSize; - - KillRing() : size(0), index(0), lastAction(actionOther) { - theRing.reserve(capacity); - } - - void kill(const char32_t* text, int textLen, bool forward) { - if (textLen == 0) { - return; - } - Utf32String killedText(text, textLen); - if (lastAction == actionKill && size > 0) { - int slot = indexToSlot[0]; - int currentLen = static_cast<int>(theRing[slot].length()); - int resultLen = currentLen + textLen; - Utf32String temp(resultLen + 1); - if (forward) { - memcpy(temp.get(), theRing[slot].get(), currentLen * sizeof(char32_t)); - memcpy(&temp[currentLen], killedText.get(), textLen * sizeof(char32_t)); - } else { - memcpy(temp.get(), killedText.get(), textLen * sizeof(char32_t)); - memcpy(&temp[textLen], theRing[slot].get(), - currentLen * sizeof(char32_t)); - } - temp[resultLen] = 0; - temp.initFromBuffer(); - theRing[slot] = temp; - } else { - if (size < capacity) { - if (size > 0) { - memmove(&indexToSlot[1], &indexToSlot[0], size); - } - indexToSlot[0] = size; - size++; - theRing.push_back(killedText); - } else { - int slot = indexToSlot[capacity - 1]; - theRing[slot] = killedText; - memmove(&indexToSlot[1], &indexToSlot[0], capacity - 1); - indexToSlot[0] = slot; - } - index = 0; - } - } - - Utf32String* yank() { return (size > 0) ? &theRing[indexToSlot[index]] : 0; } - - Utf32String* yankPop() { - if (size == 0) { - return 0; - } - ++index; - if (index == size) { - index = 0; - } - return &theRing[indexToSlot[index]]; - } -}; - -class InputBuffer { - char32_t* buf32; // input buffer - char* charWidths; // character widths from mk_wcwidth() - int buflen; // buffer size in characters - int len; // length of text in input buffer - int pos; // character position in buffer ( 0 <= pos <= len ) - - void clearScreen(PromptBase& pi); - int incrementalHistorySearch(PromptBase& pi, int startChar); - int completeLine(PromptBase& pi); - void refreshLine(PromptBase& pi); - - public: - InputBuffer(char32_t* buffer, char* widthArray, int bufferLen) - : buf32(buffer), - charWidths(widthArray), - buflen(bufferLen - 1), - len(0), - pos(0) { - buf32[0] = 0; - } - void preloadBuffer(const char* preloadText) { - size_t ucharCount = 0; - copyString8to32(buf32, buflen + 1, ucharCount, preloadText); - recomputeCharacterWidths(buf32, charWidths, static_cast<int>(ucharCount)); - len = static_cast<int>(ucharCount); - pos = static_cast<int>(ucharCount); - } - int getInputLine(PromptBase& pi); - int length(void) const { return len; } -}; - -// Special codes for keyboard input: -// -// Between Windows and the various Linux "terminal" programs, there is some -// pretty diverse behavior in the "scan codes" and escape sequences we are -// presented with. So ... we'll translate them all into our own pidgin -// pseudocode, trying to stay out of the way of UTF-8 and international -// characters. Here's the general plan. -// -// "User input keystrokes" (key chords, whatever) will be encoded as a single -// value. -// The low 21 bits are reserved for Unicode characters. Popular function-type -// keys -// get their own codes in the range 0x10200000 to (if needed) 0x1FE00000, -// currently -// just arrow keys, Home, End and Delete. Keypresses with Ctrl get ORed with -// 0x20000000, with Alt get ORed with 0x40000000. So, Ctrl+Alt+Home is encoded -// as 0x20000000 + 0x40000000 + 0x10A00000 == 0x70A00000. To keep things -// complicated, -// the Alt key is equivalent to prefixing the keystroke with ESC, so ESC -// followed by -// D is treated the same as Alt + D ... we'll just use Emacs terminology and -// call -// this "Meta". So, we will encode both ESC followed by D and Alt held down -// while D -// is pressed the same, as Meta-D, encoded as 0x40000064. -// -// Here are the definitions of our component constants: -// -// Maximum unsigned 32-bit value = 0xFFFFFFFF; // For reference, max 32-bit -// value -// Highest allocated Unicode char = 0x001FFFFF; // For reference, max -// Unicode value -static const int META = 0x40000000; // Meta key combination -static const int CTRL = 0x20000000; // Ctrl key combination -// static const int SPECIAL_KEY = 0x10000000; // Common bit for all special -// keys -static const int UP_ARROW_KEY = 0x10200000; // Special keys -static const int DOWN_ARROW_KEY = 0x10400000; -static const int RIGHT_ARROW_KEY = 0x10600000; -static const int LEFT_ARROW_KEY = 0x10800000; -static const int HOME_KEY = 0x10A00000; -static const int END_KEY = 0x10C00000; -static const int DELETE_KEY = 0x10E00000; -static const int PAGE_UP_KEY = 0x11000000; -static const int PAGE_DOWN_KEY = 0x11200000; - -static const char* unsupported_term[] = {"dumb", "cons25", "emacs", NULL}; -static linenoiseCompletionCallback* completionCallback = NULL; - -#ifdef _WIN32 -static HANDLE console_in, console_out; -static DWORD oldMode; -static WORD oldDisplayAttribute; -#else -static struct termios orig_termios; /* in order to restore at exit */ -#endif - -static KillRing killRing; - -static int rawmode = 0; /* for atexit() function to check if restore is needed*/ -static int atexit_registered = 0; /* register atexit just 1 time */ -static int historyMaxLen = LINENOISE_DEFAULT_HISTORY_MAX_LEN; -static int historyLen = 0; -static int historyIndex = 0; -static char8_t** history = NULL; - -// used to emulate Windows command prompt on down-arrow after a recall -// we use -2 as our "not set" value because we add 1 to the previous index on -// down-arrow, -// and zero is a valid index (so -1 is a valid "previous index") -static int historyPreviousIndex = -2; -static bool historyRecallMostRecent = false; - -static void linenoiseAtExit(void); - -static bool isUnsupportedTerm(void) { - char* term = getenv("TERM"); - if (term == NULL) return false; - for (int j = 0; unsupported_term[j]; ++j) - if (!strcasecmp(term, unsupported_term[j])) { - return true; - } - return false; -} - -static void beep() { - fprintf(stderr, "\x7"); // ctrl-G == bell/beep - fflush(stderr); -} - -void linenoiseHistoryFree(void) { - if (history) { - for (int j = 0; j < historyLen; ++j) free(history[j]); - historyLen = 0; - free(history); - history = 0; - } -} - -static int enableRawMode(void) { -#ifdef _WIN32 - if (!console_in) { - console_in = GetStdHandle(STD_INPUT_HANDLE); - console_out = GetStdHandle(STD_OUTPUT_HANDLE); - - GetConsoleMode(console_in, &oldMode); - SetConsoleMode(console_in, oldMode & - ~(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | - ENABLE_PROCESSED_INPUT)); - } - return 0; -#else - struct termios raw; - - if (!isatty(STDIN_FILENO)) goto fatal; - if (!atexit_registered) { - atexit(linenoiseAtExit); - atexit_registered = 1; - } - if (tcgetattr(0, &orig_termios) == -1) goto fatal; - - raw = orig_termios; /* modify the original mode */ - /* input modes: no break, no CR to NL, no parity check, no strip char, - * no start/stop output control. */ - raw.c_iflag &= ~(BRKINT | ICRNL | INPCK | ISTRIP | IXON); - /* output modes - disable post processing */ - // this is wrong, we don't want raw output, it turns newlines into straight - // linefeeds - // raw.c_oflag &= ~(OPOST); - /* control modes - set 8 bit chars */ - raw.c_cflag |= (CS8); - /* local modes - echoing off, canonical off, no extended functions, - * no signal chars (^Z,^C) */ - raw.c_lflag &= ~(ECHO | ICANON | IEXTEN | ISIG); - /* control chars - set return condition: min number of bytes and timer. - * We want read to return every single byte, without timeout. */ - raw.c_cc[VMIN] = 1; - raw.c_cc[VTIME] = 0; /* 1 byte, no timer */ - - /* put terminal in raw mode after flushing */ - if (tcsetattr(0, TCSADRAIN, &raw) < 0) goto fatal; - rawmode = 1; - return 0; - -fatal: - errno = ENOTTY; - return -1; -#endif -} - -static void disableRawMode(void) { -#ifdef _WIN32 - SetConsoleMode(console_in, oldMode); - console_in = 0; - console_out = 0; -#else - if (rawmode && tcsetattr(0, TCSADRAIN, &orig_termios) != -1) rawmode = 0; -#endif -} - -// At exit we'll try to fix the terminal to the initial conditions -static void linenoiseAtExit(void) { disableRawMode(); } - -static int getScreenColumns(void) { - int cols; -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf); - cols = inf.dwSize.X; -#else - struct winsize ws; - cols = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 80 : ws.ws_col; -#endif - // cols is 0 in certain circumstances like inside debugger, which creates - // further issues - return (cols > 0) ? cols : 80; -} - -static int getScreenRows(void) { - int rows; -#ifdef _WIN32 - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &inf); - rows = 1 + inf.srWindow.Bottom - inf.srWindow.Top; -#else - struct winsize ws; - rows = (ioctl(1, TIOCGWINSZ, &ws) == -1) ? 24 : ws.ws_row; -#endif - return (rows > 0) ? rows : 24; -} - -static void setDisplayAttribute(bool enhancedDisplay, bool error) { -#ifdef _WIN32 - if (enhancedDisplay) { - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - oldDisplayAttribute = inf.wAttributes; - BYTE oldLowByte = oldDisplayAttribute & 0xFF; - BYTE newLowByte; - switch (oldLowByte) { - case 0x07: - // newLowByte = FOREGROUND_BLUE | FOREGROUND_INTENSITY; // too dim - // newLowByte = FOREGROUND_BLUE; // even dimmer - newLowByte = FOREGROUND_BLUE | - FOREGROUND_GREEN; // most similar to xterm appearance - break; - case 0x70: - newLowByte = BACKGROUND_BLUE | BACKGROUND_INTENSITY; - break; - default: - newLowByte = oldLowByte ^ 0xFF; // default to inverse video - break; - } - inf.wAttributes = (inf.wAttributes & 0xFF00) | newLowByte; - SetConsoleTextAttribute(console_out, inf.wAttributes); - } else { - SetConsoleTextAttribute(console_out, oldDisplayAttribute); - } -#else - if (enhancedDisplay) { - char const* p = (error ? "\x1b[1;31m" : "\x1b[1;34m"); - if (write(1, p, 7) == -1) - return; /* bright blue (visible with both B&W bg) */ - } else { - if (write(1, "\x1b[0m", 4) == -1) return; /* reset */ - } -#endif -} - -/** - * Display the dynamic incremental search prompt and the current user input - * line. - * @param pi PromptBase struct holding information about the prompt and our - * screen position - * @param buf32 input buffer to be displayed - * @param len count of characters in the buffer - * @param pos current cursor position within the buffer (0 <= pos <= len) - */ -static void dynamicRefresh(PromptBase& pi, char32_t* buf32, int len, int pos) { - // calculate the position of the end of the prompt - int xEndOfPrompt, yEndOfPrompt; - calculateScreenPosition(0, 0, pi.promptScreenColumns, pi.promptChars, - xEndOfPrompt, yEndOfPrompt); - pi.promptIndentation = xEndOfPrompt; - - // calculate the position of the end of the input line - int xEndOfInput, yEndOfInput; - calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns, - calculateColumnPosition(buf32, len), xEndOfInput, - yEndOfInput); - - // calculate the desired position of the cursor - int xCursorPos, yCursorPos; - calculateScreenPosition(xEndOfPrompt, yEndOfPrompt, pi.promptScreenColumns, - calculateColumnPosition(buf32, pos), xCursorPos, - yCursorPos); - -#ifdef _WIN32 - // position at the start of the prompt, clear to end of previous input - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = 0; - inf.dwCursorPosition.Y -= pi.promptCursorRowOffset /*- pi.promptExtraLines*/; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - DWORD count; - FillConsoleOutputCharacterA(console_out, ' ', - pi.promptPreviousLen + pi.promptPreviousInputLen, - inf.dwCursorPosition, &count); - pi.promptPreviousLen = pi.promptIndentation; - pi.promptPreviousInputLen = len; - - // display the prompt - if (!pi.write()) return; - - // display the input line - if (write32(1, buf32, len) == -1) return; - - // position the cursor - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = xCursorPos; // 0-based on Win32 - inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); -#else // _WIN32 - char seq[64]; - int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position at the start of the prompt, clear to end of screen - snprintf(seq, sizeof seq, "\x1b[1G\x1b[J"); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; - - // display the prompt - if (!pi.write()) return; - - // display the input line - if (write32(1, buf32, len) == -1) return; - - // we have to generate our own newline on line wrap - if (xEndOfInput == 0 && yEndOfInput > 0) - if (write(1, "\n", 1) == -1) return; - - // position the cursor - cursorRowMovement = yEndOfInput - yCursorPos; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position the cursor within the line - snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; -#endif - - pi.promptCursorRowOffset = - pi.promptExtraLines + yCursorPos; // remember row for next pass -} - -/** - * Refresh the user's input line: the prompt is already onscreen and is not - * redrawn here - * @param pi PromptBase struct holding information about the prompt and our - * screen position - */ -void InputBuffer::refreshLine(PromptBase& pi) { - // check for a matching brace/bracket/paren, remember its position if found - int highlight = -1; - bool indicateError = false; - if (pos < len) { - /* this scans for a brace matching buf32[pos] to highlight */ - unsigned char part1, part2; - int scanDirection = 0; - if (strchr("}])", buf32[pos])) { - scanDirection = -1; /* backwards */ - if (buf32[pos] == '}') { - part1 = '}'; part2 = '{'; - } else if (buf32[pos] == ']') { - part1 = ']'; part2 = '['; - } else { - part1 = ')'; part2 = '('; - } - } - else if (strchr("{[(", buf32[pos])) { - scanDirection = 1; /* forwards */ - if (buf32[pos] == '{') { - //part1 = '{'; part2 = '}'; - part1 = '}'; part2 = '{'; - } else if (buf32[pos] == '[') { - //part1 = '['; part2 = ']'; - part1 = ']'; part2 = '['; - } else { - //part1 = '('; part2 = ')'; - part1 = ')'; part2 = '('; - } - } - - if (scanDirection) { - int unmatched = scanDirection; - int unmatchedOther = 0; - for (int i = pos + scanDirection; i >= 0 && i < len; i += scanDirection) { - /* TODO: the right thing when inside a string */ - if (strchr("}])", buf32[i])) { - if (buf32[i] == part1) { - --unmatched; - } else { - --unmatchedOther; - } - } else if (strchr("{[(", buf32[i])) { - if (buf32[i] == part2) { - ++unmatched; - } else { - ++unmatchedOther; - } - } -/* - if (strchr("}])", buf32[i])) - --unmatched; - else if (strchr("{[(", buf32[i])) - ++unmatched; -*/ - if (unmatched == 0) { - highlight = i; - indicateError = (unmatchedOther != 0); - break; - } - } - } - } - - // calculate the position of the end of the input line - int xEndOfInput, yEndOfInput; - calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns, - calculateColumnPosition(buf32, len), xEndOfInput, - yEndOfInput); - - // calculate the desired position of the cursor - int xCursorPos, yCursorPos; - calculateScreenPosition(pi.promptIndentation, 0, pi.promptScreenColumns, - calculateColumnPosition(buf32, pos), xCursorPos, - yCursorPos); - -#ifdef _WIN32 - // position at the end of the prompt, clear to end of previous input - CONSOLE_SCREEN_BUFFER_INFO inf; - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = pi.promptIndentation; // 0-based on Win32 - inf.dwCursorPosition.Y -= pi.promptCursorRowOffset - pi.promptExtraLines; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); - DWORD count; - if (len < pi.promptPreviousInputLen) - FillConsoleOutputCharacterA(console_out, ' ', pi.promptPreviousInputLen, - inf.dwCursorPosition, &count); - pi.promptPreviousInputLen = len; - - // display the input line - if (highlight == -1) { - if (write32(1, buf32, len) == -1) return; - } else { - if (write32(1, buf32, highlight) == -1) return; - setDisplayAttribute(true, indicateError); /* bright blue (visible with both B&W bg) */ - if (write32(1, &buf32[highlight], 1) == -1) return; - setDisplayAttribute(false, indicateError); - if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return; - } - - // position the cursor - GetConsoleScreenBufferInfo(console_out, &inf); - inf.dwCursorPosition.X = xCursorPos; // 0-based on Win32 - inf.dwCursorPosition.Y -= yEndOfInput - yCursorPos; - SetConsoleCursorPosition(console_out, inf.dwCursorPosition); -#else // _WIN32 - char seq[64]; - int cursorRowMovement = pi.promptCursorRowOffset - pi.promptExtraLines; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position at the end of the prompt, clear to end of screen - snprintf(seq, sizeof seq, "\x1b[%dG\x1b[J", - pi.promptIndentation + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; - - if (highlight == -1) { // write unhighlighted text - if (write32(1, buf32, len) == -1) return; - } else { // highlight the matching brace/bracket/parenthesis - if (write32(1, buf32, highlight) == -1) return; - setDisplayAttribute(true, indicateError); - if (write32(1, &buf32[highlight], 1) == -1) return; - setDisplayAttribute(false, indicateError); - if (write32(1, buf32 + highlight + 1, len - highlight - 1) == -1) return; - } - - // we have to generate our own newline on line wrap - if (xEndOfInput == 0 && yEndOfInput > 0) - if (write(1, "\n", 1) == -1) return; - - // position the cursor - cursorRowMovement = yEndOfInput - yCursorPos; - if (cursorRowMovement > 0) { // move the cursor up as required - snprintf(seq, sizeof seq, "\x1b[%dA", cursorRowMovement); - if (write(1, seq, strlen(seq)) == -1) return; - } - // position the cursor within the line - snprintf(seq, sizeof seq, "\x1b[%dG", xCursorPos + 1); // 1-based on VT100 - if (write(1, seq, strlen(seq)) == -1) return; -#endif - - pi.promptCursorRowOffset = - pi.promptExtraLines + yCursorPos; // remember row for next pass -} - -#ifndef _WIN32 - -/** - * Read a UTF-8 sequence from the non-Windows keyboard and return the Unicode - * (char32_t) character it - * encodes - * - * @return char32_t Unicode character - */ -static char32_t readUnicodeCharacter(void) { - static char8_t utf8String[5]; - static size_t utf8Count = 0; - while (true) { - char8_t c; - - /* Continue reading if interrupted by signal. */ - ssize_t nread; - do { - nread = read(0, &c, 1); - } while ((nread == -1) && (errno == EINTR)); - - if (nread <= 0) return 0; - if (c <= 0x7F) { // short circuit ASCII - utf8Count = 0; - return c; - } else if (utf8Count < sizeof(utf8String) - 1) { - utf8String[utf8Count++] = c; - utf8String[utf8Count] = 0; - char32_t unicodeChar[2]; - size_t ucharCount; - ConversionResult res = - copyString8to32(unicodeChar, 2, ucharCount, utf8String); - if (res == conversionOK && ucharCount) { - utf8Count = 0; - return unicodeChar[0]; - } - } else { - utf8Count = - 0; // this shouldn't happen: got four bytes but no UTF-8 character - } - } -} - -namespace EscapeSequenceProcessing { // move these out of global namespace - -// This chunk of code does parsing of the escape sequences sent by various Linux -// terminals. -// -// It handles arrow keys, Home, End and Delete keys by interpreting the -// sequences sent by -// gnome terminal, xterm, rxvt, konsole, aterm and yakuake including the Alt and -// Ctrl key -// combinations that are understood by linenoise. -// -// The parsing uses tables, a bunch of intermediate dispatch routines and a -// doDispatch -// loop that reads the tables and sends control to "deeper" routines to continue -// the -// parsing. The starting call to doDispatch( c, initialDispatch ) will -// eventually return -// either a character (with optional CTRL and META bits set), or -1 if parsing -// fails, or -// zero if an attempt to read from the keyboard fails. -// -// This is rather sloppy escape sequence processing, since we're not paying -// attention to what the -// actual TERM is set to and are processing all key sequences for all terminals, -// but it works with -// the most common keystrokes on the most common terminals. It's intricate, but -// the nested 'if' -// statements required to do it directly would be worse. This way has the -// advantage of allowing -// changes and extensions without having to touch a lot of code. - -// This is a typedef for the routine called by doDispatch(). It takes the -// current character -// as input, does any required processing including reading more characters and -// calling other -// dispatch routines, then eventually returns the final (possibly extended or -// special) character. -// -typedef char32_t (*CharacterDispatchRoutine)(char32_t); - -// This structure is used by doDispatch() to hold a list of characters to test -// for and -// a list of routines to call if the character matches. The dispatch routine -// list is one -// longer than the character list; the final entry is used if no character -// matches. -// -struct CharacterDispatch { - unsigned int len; // length of the chars list - const char* chars; // chars to test - CharacterDispatchRoutine* dispatch; // array of routines to call -}; - -// This dispatch routine is given a dispatch table and then farms work out to -// routines -// listed in the table based on the character it is called with. The dispatch -// routines can -// read more input characters to decide what should eventually be returned. -// Eventually, -// a called routine returns either a character or -1 to indicate parsing -// failure. -// -static char32_t doDispatch(char32_t c, CharacterDispatch& dispatchTable) { - for (unsigned int i = 0; i < dispatchTable.len; ++i) { - if (static_cast<unsigned char>(dispatchTable.chars[i]) == c) { - return dispatchTable.dispatch[i](c); - } - } - return dispatchTable.dispatch[dispatchTable.len](c); -} - -static char32_t thisKeyMetaCtrl = - 0; // holds pre-set Meta and/or Ctrl modifiers - -// Final dispatch routines -- return something -// -static char32_t normalKeyRoutine(char32_t c) { return thisKeyMetaCtrl | c; } -static char32_t upArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | UP_ARROW_KEY; -} -static char32_t downArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | DOWN_ARROW_KEY; -} -static char32_t rightArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | RIGHT_ARROW_KEY; -} -static char32_t leftArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | LEFT_ARROW_KEY; -} -static char32_t homeKeyRoutine(char32_t) { return thisKeyMetaCtrl | HOME_KEY; } -static char32_t endKeyRoutine(char32_t) { return thisKeyMetaCtrl | END_KEY; } -static char32_t pageUpKeyRoutine(char32_t) { - return thisKeyMetaCtrl | PAGE_UP_KEY; -} -static char32_t pageDownKeyRoutine(char32_t) { - return thisKeyMetaCtrl | PAGE_DOWN_KEY; -} -static char32_t deleteCharRoutine(char32_t) { - return thisKeyMetaCtrl | ctrlChar('H'); -} // key labeled Backspace -static char32_t deleteKeyRoutine(char32_t) { - return thisKeyMetaCtrl | DELETE_KEY; -} // key labeled Delete -static char32_t ctrlUpArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | UP_ARROW_KEY; -} -static char32_t ctrlDownArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | DOWN_ARROW_KEY; -} -static char32_t ctrlRightArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | RIGHT_ARROW_KEY; -} -static char32_t ctrlLeftArrowKeyRoutine(char32_t) { - return thisKeyMetaCtrl | CTRL | LEFT_ARROW_KEY; -} -static char32_t escFailureRoutine(char32_t) { - beep(); - return -1; -} - -// Handle ESC [ 1 ; 3 (or 5) <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket1Semicolon3or5Routines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, escFailureRoutine}; -static CharacterDispatch escLeftBracket1Semicolon3or5Dispatch = { - 4, "ABCD", escLeftBracket1Semicolon3or5Routines}; - -// Handle ESC [ 1 ; <more stuff> escape sequences -// -static char32_t escLeftBracket1Semicolon3Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - thisKeyMetaCtrl |= META; - return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch); -} -static char32_t escLeftBracket1Semicolon5Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - thisKeyMetaCtrl |= CTRL; - return doDispatch(c, escLeftBracket1Semicolon3or5Dispatch); -} -static CharacterDispatchRoutine escLeftBracket1SemicolonRoutines[] = { - escLeftBracket1Semicolon3Routine, escLeftBracket1Semicolon5Routine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket1SemicolonDispatch = { - 2, "35", escLeftBracket1SemicolonRoutines}; - -// Handle ESC [ 1 <more stuff> escape sequences -// -static char32_t escLeftBracket1SemicolonRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket1SemicolonDispatch); -} -static CharacterDispatchRoutine escLeftBracket1Routines[] = { - homeKeyRoutine, escLeftBracket1SemicolonRoutine, escFailureRoutine}; -static CharacterDispatch escLeftBracket1Dispatch = {2, "~;", - escLeftBracket1Routines}; - -// Handle ESC [ 3 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket3Routines[] = {deleteKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket3Dispatch = {1, "~", - escLeftBracket3Routines}; - -// Handle ESC [ 4 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket4Routines[] = {endKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket4Dispatch = {1, "~", - escLeftBracket4Routines}; - -// Handle ESC [ 5 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket5Routines[] = {pageUpKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket5Dispatch = {1, "~", - escLeftBracket5Routines}; - -// Handle ESC [ 6 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket6Routines[] = {pageDownKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket6Dispatch = {1, "~", - escLeftBracket6Routines}; - -// Handle ESC [ 7 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket7Routines[] = {homeKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket7Dispatch = {1, "~", - escLeftBracket7Routines}; - -// Handle ESC [ 8 <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracket8Routines[] = {endKeyRoutine, - escFailureRoutine}; -static CharacterDispatch escLeftBracket8Dispatch = {1, "~", - escLeftBracket8Routines}; - -// Handle ESC [ <digit> escape sequences -// -static char32_t escLeftBracket0Routine(char32_t c) { - return escFailureRoutine(c); -} -static char32_t escLeftBracket1Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket1Dispatch); -} -static char32_t escLeftBracket2Routine(char32_t c) { - return escFailureRoutine(c); // Insert key, unused -} -static char32_t escLeftBracket3Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket3Dispatch); -} -static char32_t escLeftBracket4Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket4Dispatch); -} -static char32_t escLeftBracket5Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket5Dispatch); -} -static char32_t escLeftBracket6Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket6Dispatch); -} -static char32_t escLeftBracket7Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket7Dispatch); -} -static char32_t escLeftBracket8Routine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracket8Dispatch); -} -static char32_t escLeftBracket9Routine(char32_t c) { - return escFailureRoutine(c); -} - -// Handle ESC [ <more stuff> escape sequences -// -static CharacterDispatchRoutine escLeftBracketRoutines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine, - escLeftBracket0Routine, escLeftBracket1Routine, escLeftBracket2Routine, - escLeftBracket3Routine, escLeftBracket4Routine, escLeftBracket5Routine, - escLeftBracket6Routine, escLeftBracket7Routine, escLeftBracket8Routine, - escLeftBracket9Routine, escFailureRoutine}; -static CharacterDispatch escLeftBracketDispatch = {16, "ABCDHF0123456789", - escLeftBracketRoutines}; - -// Handle ESC O <char> escape sequences -// -static CharacterDispatchRoutine escORoutines[] = { - upArrowKeyRoutine, downArrowKeyRoutine, rightArrowKeyRoutine, - leftArrowKeyRoutine, homeKeyRoutine, endKeyRoutine, - ctrlUpArrowKeyRoutine, ctrlDownArrowKeyRoutine, ctrlRightArrowKeyRoutine, - ctrlLeftArrowKeyRoutine, escFailureRoutine}; -static CharacterDispatch escODispatch = {10, "ABCDHFabcd", escORoutines}; - -// Initial ESC dispatch -- could be a Meta prefix or the start of an escape -// sequence -// -static char32_t escLeftBracketRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escLeftBracketDispatch); -} -static char32_t escORoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escODispatch); -} -static char32_t setMetaRoutine(char32_t c); // need forward reference -static CharacterDispatchRoutine escRoutines[] = {escLeftBracketRoutine, - escORoutine, setMetaRoutine}; -static CharacterDispatch escDispatch = {2, "[O", escRoutines}; - -// Initial dispatch -- we are not in the middle of anything yet -// -static char32_t escRoutine(char32_t c) { - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escDispatch); -} -static CharacterDispatchRoutine initialRoutines[] = { - escRoutine, deleteCharRoutine, normalKeyRoutine}; -static CharacterDispatch initialDispatch = {2, "\x1B\x7F", initialRoutines}; - -// Special handling for the ESC key because it does double duty -// -static char32_t setMetaRoutine(char32_t c) { - thisKeyMetaCtrl = META; - if (c == 0x1B) { // another ESC, stay in ESC processing mode - c = readUnicodeCharacter(); - if (c == 0) return 0; - return doDispatch(c, escDispatch); - } - return doDispatch(c, initialDispatch); -} - -} // namespace EscapeSequenceProcessing // move these out of global namespace - -#endif // #ifndef _WIN32 - -// linenoiseReadChar -- read a keystroke or keychord from the keyboard, and -// translate it -// into an encoded "keystroke". When convenient, extended keys are translated -// into their -// simpler Emacs keystrokes, so an unmodified "left arrow" becomes Ctrl-B. -// -// A return value of zero means "no input available", and a return value of -1 -// means "invalid key". -// -static char32_t linenoiseReadChar(void) { -#ifdef _WIN32 - - INPUT_RECORD rec; - DWORD count; - int modifierKeys = 0; - bool escSeen = false; - while (true) { - ReadConsoleInputW(console_in, &rec, 1, &count); -#if 0 // helper for debugging keystrokes, display info in the debug "Output" - // window in the debugger - { - if ( rec.EventType == KEY_EVENT ) { - //if ( rec.Event.KeyEvent.uChar.UnicodeChar ) { - char buf[1024]; - sprintf( - buf, - "Unicode character 0x%04X, repeat count %d, virtual keycode 0x%04X, " - "virtual scancode 0x%04X, key %s%s%s%s%s\n", - rec.Event.KeyEvent.uChar.UnicodeChar, - rec.Event.KeyEvent.wRepeatCount, - rec.Event.KeyEvent.wVirtualKeyCode, - rec.Event.KeyEvent.wVirtualScanCode, - rec.Event.KeyEvent.bKeyDown ? "down" : "up", - (rec.Event.KeyEvent.dwControlKeyState & LEFT_CTRL_PRESSED) ? - " L-Ctrl" : "", - (rec.Event.KeyEvent.dwControlKeyState & RIGHT_CTRL_PRESSED) ? - " R-Ctrl" : "", - (rec.Event.KeyEvent.dwControlKeyState & LEFT_ALT_PRESSED) ? - " L-Alt" : "", - (rec.Event.KeyEvent.dwControlKeyState & RIGHT_ALT_PRESSED) ? - " R-Alt" : "" - ); - OutputDebugStringA( buf ); - //} - } - } -#endif - if (rec.EventType != KEY_EVENT) { - continue; - } - // Windows provides for entry of characters that are not on your keyboard by - // sending the - // Unicode characters as a "key up" with virtual keycode 0x12 (VK_MENU == - // Alt key) ... - // accept these characters, otherwise only process characters on "key down" - if (!rec.Event.KeyEvent.bKeyDown && - rec.Event.KeyEvent.wVirtualKeyCode != VK_MENU) { - continue; - } - modifierKeys = 0; - // AltGr is encoded as ( LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED ), so don't - // treat this - // combination as either CTRL or META we just turn off those two bits, so it - // is still - // possible to combine CTRL and/or META with an AltGr key by using - // right-Ctrl and/or - // left-Alt - if ((rec.Event.KeyEvent.dwControlKeyState & - (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) == - (LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED)) { - rec.Event.KeyEvent.dwControlKeyState &= - ~(LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED); - } - if (rec.Event.KeyEvent.dwControlKeyState & - (RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED)) { - modifierKeys |= CTRL; - } - if (rec.Event.KeyEvent.dwControlKeyState & - (RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED)) { - modifierKeys |= META; - } - if (escSeen) { - modifierKeys |= META; - } - if (rec.Event.KeyEvent.uChar.UnicodeChar == 0) { - switch (rec.Event.KeyEvent.wVirtualKeyCode) { - case VK_LEFT: - return modifierKeys | LEFT_ARROW_KEY; - case VK_RIGHT: - return modifierKeys | RIGHT_ARROW_KEY; - case VK_UP: - return modifierKeys | UP_ARROW_KEY; - case VK_DOWN: - return modifierKeys | DOWN_ARROW_KEY; - case VK_DELETE: - return modifierKeys | DELETE_KEY; - case VK_HOME: - return modifierKeys | HOME_KEY; - case VK_END: - return modifierKeys | END_KEY; - case VK_PRIOR: - return modifierKeys | PAGE_UP_KEY; - case VK_NEXT: - return modifierKeys | PAGE_DOWN_KEY; - default: - continue; // in raw mode, ReadConsoleInput shows shift, ctrl ... - } // ... ignore them - } else if (rec.Event.KeyEvent.uChar.UnicodeChar == - ctrlChar('[')) { // ESC, set flag for later - escSeen = true; - continue; - } else { - // we got a real character, return it - return modifierKeys | rec.Event.KeyEvent.uChar.UnicodeChar; - } - } - -#else - char32_t c; - c = readUnicodeCharacter(); - if (c == 0) return 0; - -// If _DEBUG_LINUX_KEYBOARD is set, then ctrl-^ puts us into a keyboard -// debugging mode -// where we print out decimal and decoded values for whatever the "terminal" -// program -// gives us on different keystrokes. Hit ctrl-C to exit this mode. -// -#define _DEBUG_LINUX_KEYBOARD -#if defined(_DEBUG_LINUX_KEYBOARD) - if (c == ctrlChar('^')) { // ctrl-^, special debug mode, prints all keys hit, - // ctrl-C to get out - printf( - "\nEntering keyboard debugging mode (on ctrl-^), press ctrl-C to exit " - "this mode\n"); - while (true) { - unsigned char keys[10]; - int ret = read(0, keys, 10); - - if (ret <= 0) { - printf("\nret: %d\n", ret); - } - for (int i = 0; i < ret; ++i) { - char32_t key = static_cast<char32_t>(keys[i]); - char* friendlyTextPtr; - char friendlyTextBuf[10]; - const char* prefixText = (key < 0x80) ? "" : "0x80+"; - char32_t keyCopy = (key < 0x80) ? key : key - 0x80; - if (keyCopy >= '!' && keyCopy <= '~') { // printable - friendlyTextBuf[0] = '\''; - friendlyTextBuf[1] = keyCopy; - friendlyTextBuf[2] = '\''; - friendlyTextBuf[3] = 0; - friendlyTextPtr = friendlyTextBuf; - } else if (keyCopy == ' ') { - friendlyTextPtr = const_cast<char*>("space"); - } else if (keyCopy == 27) { - friendlyTextPtr = const_cast<char*>("ESC"); - } else if (keyCopy == 0) { - friendlyTextPtr = const_cast<char*>("NUL"); - } else if (keyCopy == 127) { - friendlyTextPtr = const_cast<char*>("DEL"); - } else { - friendlyTextBuf[0] = '^'; - friendlyTextBuf[1] = keyCopy + 0x40; - friendlyTextBuf[2] = 0; - friendlyTextPtr = friendlyTextBuf; - } - printf("%d x%02X (%s%s) ", key, key, prefixText, friendlyTextPtr); - } - printf("\x1b[1G\n"); // go to first column of new line - - // drop out of this loop on ctrl-C - if (keys[0] == ctrlChar('C')) { - printf("Leaving keyboard debugging mode (on ctrl-C)\n"); - fflush(stdout); - return -2; - } - } - } -#endif // _DEBUG_LINUX_KEYBOARD - - EscapeSequenceProcessing::thisKeyMetaCtrl = - 0; // no modifiers yet at initialDispatch - return EscapeSequenceProcessing::doDispatch( - c, EscapeSequenceProcessing::initialDispatch); -#endif // #_WIN32 -} - -/** - * Free memory used in a recent command completion session - * - * @param lc pointer to a linenoiseCompletions struct - */ -static void freeCompletions(linenoiseCompletions* lc) { - lc->completionStrings.clear(); -} - -/** - * convert {CTRL + 'A'}, {CTRL + 'a'} and {CTRL + ctrlChar( 'A' )} into - * ctrlChar( 'A' ) - * leave META alone - * - * @param c character to clean up - * @return cleaned-up character - */ -static int cleanupCtrl(int c) { - if (c & CTRL) { - int d = c & 0x1FF; - if (d >= 'a' && d <= 'z') { - c = (c + ('a' - ctrlChar('A'))) & ~CTRL; - } - if (d >= 'A' && d <= 'Z') { - c = (c + ('A' - ctrlChar('A'))) & ~CTRL; - } - if (d >= ctrlChar('A') && d <= ctrlChar('Z')) { - c = c & ~CTRL; - } - } - return c; -} - -// break characters that may precede items to be completed -static const char breakChars[] = " =+-/\\*?\"'`&<>;|@{([])}"; - -// maximum number of completions to display without asking -static const size_t completionCountCutoff = 100; - -/** - * Handle command completion, using a completionCallback() routine to provide - * possible substitutions - * This routine handles the mechanics of updating the user's input buffer with - * possible replacement - * of text as the user selects a proposed completion string, or cancels the - * completion attempt. - * @param pi PromptBase struct holding information about the prompt and our - * screen position - */ -int InputBuffer::completeLine(PromptBase& pi) { - linenoiseCompletions lc; - char32_t c = 0; - - // completionCallback() expects a parsable entity, so find the previous break - // character and - // extract a copy to parse. we also handle the case where tab is hit while - // not at end-of-line. - int startIndex = pos; - while (--startIndex >= 0) { - if (strchr(breakChars, buf32[startIndex])) { - break; - } - } - ++startIndex; - int itemLength = pos - startIndex; - Utf32String unicodeCopy(&buf32[startIndex], itemLength); - Utf8String parseItem(unicodeCopy); - - // get a list of completions - completionCallback(parseItem.get(), &lc); - - // if no completions, we are done - if (lc.completionStrings.size() == 0) { - beep(); - freeCompletions(&lc); - return 0; - } - - // at least one completion - int longestCommonPrefix = 0; - int displayLength = 0; - if (lc.completionStrings.size() == 1) { - longestCommonPrefix = static_cast<int>(lc.completionStrings[0].length()); - } else { - bool keepGoing = true; - while (keepGoing) { - for (size_t j = 0; j < lc.completionStrings.size() - 1; ++j) { - char32_t c1 = lc.completionStrings[j][longestCommonPrefix]; - char32_t c2 = lc.completionStrings[j + 1][longestCommonPrefix]; - if ((0 == c1) || (0 == c2) || (c1 != c2)) { - keepGoing = false; - break; - } - } - if (keepGoing) { - ++longestCommonPrefix; - } - } - } - if (lc.completionStrings.size() != 1) { // beep if ambiguous - beep(); - } - - // if we can extend the item, extend it and return to main loop - if (longestCommonPrefix > itemLength) { - displayLength = len + longestCommonPrefix - itemLength; - if (displayLength > buflen) { - longestCommonPrefix -= displayLength - buflen; // don't overflow buffer - displayLength = buflen; // truncate the insertion - beep(); // and make a noise - } - Utf32String displayText(displayLength + 1); - memcpy(displayText.get(), buf32, sizeof(char32_t) * startIndex); - memcpy(&displayText[startIndex], &lc.completionStrings[0][0], - sizeof(char32_t) * longestCommonPrefix); - int tailIndex = startIndex + longestCommonPrefix; - memcpy(&displayText[tailIndex], &buf32[pos], - sizeof(char32_t) * (displayLength - tailIndex + 1)); - copyString32(buf32, displayText.get(), displayLength); - pos = startIndex + longestCommonPrefix; - len = displayLength; - refreshLine(pi); - return 0; - } - - // we can't complete any further, wait for second tab - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - - // if any character other than tab, pass it to the main loop - if (c != ctrlChar('I')) { - freeCompletions(&lc); - return c; - } - - // we got a second tab, maybe show list of possible completions - bool showCompletions = true; - bool onNewLine = false; - if (lc.completionStrings.size() > completionCountCutoff) { - int savePos = - pos; // move cursor to EOL to avoid overwriting the command line - pos = len; - refreshLine(pi); - pos = savePos; - printf("\nDisplay all %u possibilities? (y or n)", - static_cast<unsigned int>(lc.completionStrings.size())); - fflush(stdout); - onNewLine = true; - while (c != 'y' && c != 'Y' && c != 'n' && c != 'N' && c != ctrlChar('C')) { - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - } - switch (c) { - case 'n': - case 'N': - showCompletions = false; - freeCompletions(&lc); - break; - case ctrlChar('C'): - showCompletions = false; - freeCompletions(&lc); - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - c = 0; - break; - } - } - - // if showing the list, do it the way readline does it - bool stopList = false; - if (showCompletions) { - int longestCompletion = 0; - for (size_t j = 0; j < lc.completionStrings.size(); ++j) { - itemLength = static_cast<int>(lc.completionStrings[j].length()); - if (itemLength > longestCompletion) { - longestCompletion = itemLength; - } - } - longestCompletion += 2; - int columnCount = pi.promptScreenColumns / longestCompletion; - if (columnCount < 1) { - columnCount = 1; - } - if (!onNewLine) { // skip this if we showed "Display all %d possibilities?" - int savePos = - pos; // move cursor to EOL to avoid overwriting the command line - pos = len; - refreshLine(pi); - pos = savePos; - } - size_t pauseRow = getScreenRows() - 1; - size_t rowCount = - (lc.completionStrings.size() + columnCount - 1) / columnCount; - for (size_t row = 0; row < rowCount; ++row) { - if (row == pauseRow) { - printf("\n--More--"); - fflush(stdout); - c = 0; - bool doBeep = false; - while (c != ' ' && c != '\r' && c != '\n' && c != 'y' && c != 'Y' && - c != 'n' && c != 'N' && c != 'q' && c != 'Q' && - c != ctrlChar('C')) { - if (doBeep) { - beep(); - } - doBeep = true; - do { - c = linenoiseReadChar(); - c = cleanupCtrl(c); - } while (c == static_cast<char32_t>(-1)); - } - switch (c) { - case ' ': - case 'y': - case 'Y': - printf("\r \r"); - pauseRow += getScreenRows() - 1; - break; - case '\r': - case '\n': - printf("\r \r"); - ++pauseRow; - break; - case 'n': - case 'N': - case 'q': - case 'Q': - printf("\r \r"); - stopList = true; - break; - case ctrlChar('C'): - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - stopList = true; - break; - } - } else { - printf("\n"); - } - if (stopList) { - break; - } - for (int column = 0; column < columnCount; ++column) { - size_t index = (column * rowCount) + row; - if (index < lc.completionStrings.size()) { - itemLength = static_cast<int>(lc.completionStrings[index].length()); - fflush(stdout); - if (write32(1, lc.completionStrings[index].get(), itemLength) == -1) - return -1; - if (((column + 1) * rowCount) + row < lc.completionStrings.size()) { - for (int k = itemLength; k < longestCompletion; ++k) { - printf(" "); - } - } - } - } - } - fflush(stdout); - freeCompletions(&lc); - } - - // display the prompt on a new line, then redisplay the input buffer - if (!stopList || c == ctrlChar('C')) { - if (write(1, "\n", 1) == -1) return 0; - } - if (!pi.write()) return 0; -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return 0; -#endif - pi.promptCursorRowOffset = pi.promptExtraLines; - refreshLine(pi); - return 0; -} - -/** - * Clear the screen ONLY (no redisplay of anything) - */ -void linenoiseClearScreen(void) { -#ifdef _WIN32 - COORD coord = {0, 0}; - CONSOLE_SCREEN_BUFFER_INFO inf; - HANDLE screenHandle = GetStdHandle(STD_OUTPUT_HANDLE); - GetConsoleScreenBufferInfo(screenHandle, &inf); - SetConsoleCursorPosition(screenHandle, coord); - DWORD count; - FillConsoleOutputCharacterA(screenHandle, ' ', inf.dwSize.X * inf.dwSize.Y, - coord, &count); -#else - if (write(1, "\x1b[H\x1b[2J", 7) <= 0) return; -#endif -} - -void InputBuffer::clearScreen(PromptBase& pi) { - linenoiseClearScreen(); - if (!pi.write()) return; -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return; -#endif - pi.promptCursorRowOffset = pi.promptExtraLines; - refreshLine(pi); -} - -/** - * Incremental history search -- take over the prompt and keyboard as the user - * types a search - * string, deletes characters from it, changes direction, and either accepts the - * found line (for - * execution orediting) or cancels. - * @param pi PromptBase struct holding information about the (old, - * static) prompt and our - * screen position - * @param startChar the character that began the search, used to set the initial - * direction - */ -int InputBuffer::incrementalHistorySearch(PromptBase& pi, int startChar) { - size_t bufferSize; - size_t ucharCount = 0; - - // if not already recalling, add the current line to the history list so we - // don't have to - // special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - bufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - copyString32to8(tempBuffer.get(), bufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - int historyLineLength = len; - int historyLinePosition = pos; - char32_t emptyBuffer[1]; - char emptyWidths[1]; - InputBuffer empty(emptyBuffer, emptyWidths, 1); - empty.refreshLine(pi); // erase the old input first - DynamicPrompt dp(pi, (startChar == ctrlChar('R')) ? -1 : 1); - - dp.promptPreviousLen = pi.promptPreviousLen; - dp.promptPreviousInputLen = pi.promptPreviousInputLen; - dynamicRefresh(dp, buf32, historyLineLength, - historyLinePosition); // draw user's text with our prompt - - // loop until we get an exit character - int c = 0; - bool keepLooping = true; - bool useSearchedLine = true; - bool searchAgain = false; - char32_t* activeHistoryLine = 0; - while (keepLooping) { - c = linenoiseReadChar(); - c = cleanupCtrl(c); // convert CTRL + <char> into normal ctrl - - switch (c) { - // these characters keep the selected text but do not execute it - case ctrlChar('A'): // ctrl-A, move cursor to start of line - case HOME_KEY: - case ctrlChar('B'): // ctrl-B, move cursor left by one character - case LEFT_ARROW_KEY: - case META + 'b': // meta-B, move cursor left by one word - case META + 'B': - case CTRL + LEFT_ARROW_KEY: - case META + LEFT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - case ctrlChar('D'): - case META + 'd': // meta-D, kill word to right of cursor - case META + 'D': - case ctrlChar('E'): // ctrl-E, move cursor to end of line - case END_KEY: - case ctrlChar('F'): // ctrl-F, move cursor right by one character - case RIGHT_ARROW_KEY: - case META + 'f': // meta-F, move cursor right by one word - case META + 'F': - case CTRL + RIGHT_ARROW_KEY: - case META + RIGHT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - case META + ctrlChar('H'): - case ctrlChar('J'): - case ctrlChar('K'): // ctrl-K, kill from cursor to end of line - case ctrlChar('M'): - case ctrlChar('N'): // ctrl-N, recall next line in history - case ctrlChar('P'): // ctrl-P, recall previous line in history - case DOWN_ARROW_KEY: - case UP_ARROW_KEY: - case ctrlChar('T'): // ctrl-T, transpose characters - case ctrlChar( - 'U'): // ctrl-U, kill all characters to the left of the cursor - case ctrlChar('W'): - case META + 'y': // meta-Y, "yank-pop", rotate popped text - case META + 'Y': - case 127: - case DELETE_KEY: - case META + '<': // start of history - case PAGE_UP_KEY: - case META + '>': // end of history - case PAGE_DOWN_KEY: - keepLooping = false; - break; - - // these characters revert the input line to its previous state - case ctrlChar('C'): // ctrl-C, abort this line - case ctrlChar('G'): - case ctrlChar('L'): // ctrl-L, clear screen and redisplay line - keepLooping = false; - useSearchedLine = false; - if (c != ctrlChar('L')) { - c = -1; // ctrl-C and ctrl-G just abort the search and do nothing - // else - } - break; - - // these characters stay in search mode and update the display - case ctrlChar('S'): - case ctrlChar('R'): - if (dp.searchTextLen == - 0) { // if no current search text, recall previous text - if (previousSearchText.length()) { - dp.updateSearchText(previousSearchText.get()); - } - } - if ((dp.direction == 1 && c == ctrlChar('R')) || - (dp.direction == -1 && c == ctrlChar('S'))) { - dp.direction = 0 - dp.direction; // reverse direction - dp.updateSearchPrompt(); // change the prompt - } else { - searchAgain = true; // same direction, search again - } - break; - -// job control is its own thing -#ifndef _WIN32 - case ctrlChar('Z'): // ctrl-Z, job control - disableRawMode(); // Returning to Linux (whatever) shell, leave raw - // mode - raise(SIGSTOP); // Break out in mid-line - enableRawMode(); // Back from Linux shell, re-enter raw mode - { - bufferSize = historyLineLength + 1; - unique_ptr<char32_t[]> tempUnicode(new char32_t[bufferSize]); - copyString8to32(tempUnicode.get(), bufferSize, ucharCount, - history[historyIndex]); - dynamicRefresh(dp, tempUnicode.get(), historyLineLength, - historyLinePosition); - } - continue; - break; -#endif - - // these characters update the search string, and hence the selected input - // line - case ctrlChar('H'): // backspace/ctrl-H, delete char to left of cursor - if (dp.searchTextLen > 0) { - unique_ptr<char32_t[]> tempUnicode(new char32_t[dp.searchTextLen]); - --dp.searchTextLen; - dp.searchText[dp.searchTextLen] = 0; - copyString32(tempUnicode.get(), dp.searchText.get(), - dp.searchTextLen); - dp.updateSearchText(tempUnicode.get()); - } else { - beep(); - } - break; - - case ctrlChar('Y'): // ctrl-Y, yank killed text - break; - - default: - if (!isControlChar(c) && c <= 0x0010FFFF) { // not an action character - unique_ptr<char32_t[]> tempUnicode( - new char32_t[dp.searchTextLen + 2]); - copyString32(tempUnicode.get(), dp.searchText.get(), - dp.searchTextLen); - tempUnicode[dp.searchTextLen] = c; - tempUnicode[dp.searchTextLen + 1] = 0; - dp.updateSearchText(tempUnicode.get()); - } else { - beep(); - } - } // switch - - // if we are staying in search mode, search now - if (keepLooping) { - bufferSize = historyLineLength + 1; - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historyIndex]); - if (dp.searchTextLen > 0) { - bool found = false; - int historySearchIndex = historyIndex; - int lineLength = static_cast<int>(ucharCount); - int lineSearchPos = historyLinePosition; - if (searchAgain) { - lineSearchPos += dp.direction; - } - searchAgain = false; - while (true) { - while ((dp.direction > 0) ? (lineSearchPos < lineLength) - : (lineSearchPos >= 0)) { - if (strncmp32(dp.searchText.get(), - &activeHistoryLine[lineSearchPos], - dp.searchTextLen) == 0) { - found = true; - break; - } - lineSearchPos += dp.direction; - } - if (found) { - historyIndex = historySearchIndex; - historyLineLength = lineLength; - historyLinePosition = lineSearchPos; - break; - } else if ((dp.direction > 0) ? (historySearchIndex < historyLen - 1) - : (historySearchIndex > 0)) { - historySearchIndex += dp.direction; - bufferSize = strlen8(history[historySearchIndex]) + 1; - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historySearchIndex]); - lineLength = static_cast<int>(ucharCount); - lineSearchPos = - (dp.direction > 0) ? 0 : (lineLength - dp.searchTextLen); - } else { - beep(); - break; - } - }; // while - } - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - bufferSize = historyLineLength + 1; - activeHistoryLine = new char32_t[bufferSize]; - copyString8to32(activeHistoryLine, bufferSize, ucharCount, - history[historyIndex]); - dynamicRefresh(dp, activeHistoryLine, historyLineLength, - historyLinePosition); // draw user's text with our prompt - } - } // while - - // leaving history search, restore previous prompt, maybe make searched line - // current - PromptBase pb; - pb.promptChars = pi.promptIndentation; - pb.promptBytes = pi.promptBytes; - Utf32String tempUnicode(pb.promptBytes + 1); - - copyString32(tempUnicode.get(), &pi.promptText[pi.promptLastLinePosition], - pb.promptBytes - pi.promptLastLinePosition); - tempUnicode.initFromBuffer(); - pb.promptText = tempUnicode; - pb.promptExtraLines = 0; - pb.promptIndentation = pi.promptIndentation; - pb.promptLastLinePosition = 0; - pb.promptPreviousInputLen = historyLineLength; - pb.promptCursorRowOffset = dp.promptCursorRowOffset; - pb.promptScreenColumns = pi.promptScreenColumns; - pb.promptPreviousLen = dp.promptChars; - if (useSearchedLine && activeHistoryLine) { - historyRecallMostRecent = true; - copyString32(buf32, activeHistoryLine, buflen + 1); - len = historyLineLength; - pos = historyLinePosition; - } - if (activeHistoryLine) { - delete[] activeHistoryLine; - activeHistoryLine = nullptr; - } - dynamicRefresh(pb, buf32, len, - pos); // redraw the original prompt with current input - pi.promptPreviousInputLen = len; - pi.promptCursorRowOffset = pi.promptExtraLines + pb.promptCursorRowOffset; - previousSearchText = - dp.searchText; // save search text for possible reuse on ctrl-R ctrl-R - return c; // pass a character or -1 back to main loop -} - -static bool isCharacterAlphanumeric(char32_t testChar) { -#ifdef _WIN32 - return (iswalnum((wint_t)testChar) != 0 ? true : false); -#else - return (iswalnum(testChar) != 0 ? true : false); -#endif -} - -#ifndef _WIN32 -static bool gotResize = false; -#endif -static int keyType = 0; - -int InputBuffer::getInputLine(PromptBase& pi) { - keyType = 0; - - // The latest history entry is always our current buffer - if (len > 0) { - size_t bufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - copyString32to8(tempBuffer.get(), bufferSize, buf32); - linenoiseHistoryAdd(tempBuffer.get()); - } else { - linenoiseHistoryAdd(""); - } - historyIndex = historyLen - 1; - historyRecallMostRecent = false; - - // display the prompt - if (!pi.write()) return -1; - -#ifndef _WIN32 - // we have to generate our own newline on line wrap on Linux - if (pi.promptIndentation == 0 && pi.promptExtraLines > 0) - if (write(1, "\n", 1) == -1) return -1; -#endif - - // the cursor starts out at the end of the prompt - pi.promptCursorRowOffset = pi.promptExtraLines; - - // kill and yank start in "other" mode - killRing.lastAction = KillRing::actionOther; - - // when history search returns control to us, we execute its terminating - // keystroke - int terminatingKeystroke = -1; - - // if there is already text in the buffer, display it first - if (len > 0) { - refreshLine(pi); - } - - // loop collecting characters, respond to line editing characters - while (true) { - int c; - if (terminatingKeystroke == -1) { - c = linenoiseReadChar(); // get a new keystroke - - keyType = 0; - if (c != 0) { - // set flag that we got some input - if (c == ctrlChar('C')) { - keyType = 1; - } else if (c == ctrlChar('D')) { - keyType = 2; - } - } - -#ifndef _WIN32 - if (c == 0 && gotResize) { - // caught a window resize event - // now redraw the prompt and line - gotResize = false; - pi.promptScreenColumns = getScreenColumns(); - dynamicRefresh(pi, buf32, len, - pos); // redraw the original prompt with current input - continue; - } -#endif - } else { - c = terminatingKeystroke; // use the terminating keystroke from search - terminatingKeystroke = -1; // clear it once we've used it - } - - c = cleanupCtrl(c); // convert CTRL + <char> into normal ctrl - - if (c == 0) { - return len; - } - - if (c == -1) { - refreshLine(pi); - continue; - } - - if (c == -2) { - if (!pi.write()) return -1; - refreshLine(pi); - continue; - } - - // ctrl-I/tab, command completion, needs to be before switch statement - if (c == ctrlChar('I') && completionCallback) { - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - - // completeLine does the actual completion and replacement - c = completeLine(pi); - - if (c < 0) // return on error - return len; - - if (c == 0) // read next character when 0 - continue; - - // deliberate fall-through here, so we use the terminating character - } - - switch (c) { - case ctrlChar('A'): // ctrl-A, move cursor to start of line - case HOME_KEY: - killRing.lastAction = KillRing::actionOther; - pos = 0; - refreshLine(pi); - break; - - case ctrlChar('B'): // ctrl-B, move cursor left by one character - case LEFT_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - --pos; - refreshLine(pi); - } - break; - - case META + 'b': // meta-B, move cursor left by one word - case META + 'B': - case CTRL + LEFT_ARROW_KEY: - case META + LEFT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('C'): // ctrl-C, abort this line - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - errno = EAGAIN; - --historyLen; - free(history[historyLen]); - // we need one last refresh with the cursor at the end of the line - // so we don't display the next prompt over the previous input line - pos = len; // pass len as pos for EOL - refreshLine(pi); - if (write(1, "^C", 2) == -1) return -1; // Display the ^C we got - return -1; - - case META + 'c': // meta-C, give word initial Cap - case META + 'C': - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - if (pos < len) { - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - if (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'a' && buf32[pos] <= 'z') { - buf32[pos] += 'A' - 'a'; - } - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') { - buf32[pos] += 'a' - 'A'; - } - ++pos; - } - refreshLine(pi); - } - break; - - // ctrl-D, delete the character under the cursor - // on an empty line, exit the shell - case ctrlChar('D'): - killRing.lastAction = KillRing::actionOther; - if (len > 0 && pos < len) { - historyRecallMostRecent = false; - memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos)); - --len; - refreshLine(pi); - } else if (len == 0) { - --historyLen; - free(history[historyLen]); - return -1; - } - break; - - case META + 'd': // meta-D, kill word to right of cursor - case META + 'D': - if (pos < len) { - historyRecallMostRecent = false; - int endingPos = pos; - while (endingPos < len && - !isCharacterAlphanumeric(buf32[endingPos])) { - ++endingPos; - } - while (endingPos < len && isCharacterAlphanumeric(buf32[endingPos])) { - ++endingPos; - } - killRing.kill(&buf32[pos], endingPos - pos, true); - memmove(buf32 + pos, buf32 + endingPos, - sizeof(char32_t) * (len - endingPos + 1)); - len -= endingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('E'): // ctrl-E, move cursor to end of line - case END_KEY: - killRing.lastAction = KillRing::actionOther; - pos = len; - refreshLine(pi); - break; - - case ctrlChar('F'): // ctrl-F, move cursor right by one character - case RIGHT_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - ++pos; - refreshLine(pi); - } - break; - - case META + 'f': // meta-F, move cursor right by one word - case META + 'F': - case CTRL + RIGHT_ARROW_KEY: - case META + RIGHT_ARROW_KEY: // Emacs allows Meta, bash & readline don't - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('H'): // backspace/ctrl-H, delete char to left of cursor - killRing.lastAction = KillRing::actionOther; - if (pos > 0) { - historyRecallMostRecent = false; - memmove(buf32 + pos - 1, buf32 + pos, - sizeof(char32_t) * (1 + len - pos)); - --pos; - --len; - refreshLine(pi); - } - break; - - // meta-Backspace, kill word to left of cursor - case META + ctrlChar('H'): - if (pos > 0) { - historyRecallMostRecent = false; - int startingPos = pos; - while (pos > 0 && !isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - while (pos > 0 && isCharacterAlphanumeric(buf32[pos - 1])) { - --pos; - } - killRing.kill(&buf32[pos], startingPos - pos, false); - memmove(buf32 + pos, buf32 + startingPos, - sizeof(char32_t) * (len - startingPos + 1)); - len -= startingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('J'): // ctrl-J/linefeed/newline, accept line - case ctrlChar('M'): // ctrl-M/return/enter - killRing.lastAction = KillRing::actionOther; - // we need one last refresh with the cursor at the end of the line - // so we don't display the next prompt over the previous input line - pos = len; // pass len as pos for EOL - refreshLine(pi); - historyPreviousIndex = historyRecallMostRecent ? historyIndex : -2; - --historyLen; - free(history[historyLen]); - return len; - - case ctrlChar('K'): // ctrl-K, kill from cursor to end of line - killRing.kill(&buf32[pos], len - pos, true); - buf32[pos] = '\0'; - len = pos; - refreshLine(pi); - killRing.lastAction = KillRing::actionKill; - historyRecallMostRecent = false; - break; - - case ctrlChar('L'): // ctrl-L, clear screen and redisplay line - clearScreen(pi); - break; - - case META + 'l': // meta-L, lowercase word - case META + 'L': - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - historyRecallMostRecent = false; - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'A' && buf32[pos] <= 'Z') { - buf32[pos] += 'a' - 'A'; - } - ++pos; - } - refreshLine(pi); - } - break; - - case ctrlChar('N'): // ctrl-N, recall next line in history - case ctrlChar('P'): // ctrl-P, recall previous line in history - case DOWN_ARROW_KEY: - case UP_ARROW_KEY: - killRing.lastAction = KillRing::actionOther; - // if not already recalling, add the current line to the history list so - // we don't - // have to special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - size_t tempBufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[tempBufferSize]); - copyString32to8(tempBuffer.get(), tempBufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - if (historyLen > 1) { - if (c == UP_ARROW_KEY) { - c = ctrlChar('P'); - } - if (historyPreviousIndex != -2 && c != ctrlChar('P')) { - historyIndex = - 1 + historyPreviousIndex; // emulate Windows down-arrow - } else { - historyIndex += (c == ctrlChar('P')) ? -1 : 1; - } - historyPreviousIndex = -2; - if (historyIndex < 0) { - historyIndex = 0; - break; - } else if (historyIndex >= historyLen) { - historyIndex = historyLen - 1; - break; - } - historyRecallMostRecent = true; - size_t ucharCount = 0; - copyString8to32(buf32, buflen, ucharCount, history[historyIndex]); - len = pos = static_cast<int>(ucharCount); - refreshLine(pi); - } - break; - - case ctrlChar('R'): // ctrl-R, reverse history search - case ctrlChar('S'): // ctrl-S, forward history search - terminatingKeystroke = incrementalHistorySearch(pi, c); - break; - - case ctrlChar('T'): // ctrl-T, transpose characters - killRing.lastAction = KillRing::actionOther; - if (pos > 0 && len > 1) { - historyRecallMostRecent = false; - size_t leftCharPos = (pos == len) ? pos - 2 : pos - 1; - char32_t aux = buf32[leftCharPos]; - buf32[leftCharPos] = buf32[leftCharPos + 1]; - buf32[leftCharPos + 1] = aux; - if (pos != len) ++pos; - refreshLine(pi); - } - break; - - case ctrlChar( - 'U'): // ctrl-U, kill all characters to the left of the cursor - if (pos > 0) { - historyRecallMostRecent = false; - killRing.kill(&buf32[0], pos, false); - len -= pos; - memmove(buf32, buf32 + pos, sizeof(char32_t) * (len + 1)); - pos = 0; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case META + 'u': // meta-U, uppercase word - case META + 'U': - killRing.lastAction = KillRing::actionOther; - if (pos < len) { - historyRecallMostRecent = false; - while (pos < len && !isCharacterAlphanumeric(buf32[pos])) { - ++pos; - } - while (pos < len && isCharacterAlphanumeric(buf32[pos])) { - if (buf32[pos] >= 'a' && buf32[pos] <= 'z') { - buf32[pos] += 'A' - 'a'; - } - ++pos; - } - refreshLine(pi); - } - break; - - // ctrl-W, kill to whitespace (not word) to left of cursor - case ctrlChar('W'): - if (pos > 0) { - historyRecallMostRecent = false; - int startingPos = pos; - while (pos > 0 && buf32[pos - 1] == ' ') { - --pos; - } - while (pos > 0 && buf32[pos - 1] != ' ') { - --pos; - } - killRing.kill(&buf32[pos], startingPos - pos, false); - memmove(buf32 + pos, buf32 + startingPos, - sizeof(char32_t) * (len - startingPos + 1)); - len -= startingPos - pos; - refreshLine(pi); - } - killRing.lastAction = KillRing::actionKill; - break; - - case ctrlChar('Y'): // ctrl-Y, yank killed text - historyRecallMostRecent = false; - { - Utf32String* restoredText = killRing.yank(); - if (restoredText) { - bool truncated = false; - size_t ucharCount = restoredText->length(); - if (ucharCount > static_cast<size_t>(buflen - len)) { - ucharCount = buflen - len; - truncated = true; - } - memmove(buf32 + pos + ucharCount, buf32 + pos, - sizeof(char32_t) * (len - pos + 1)); - memmove(buf32 + pos, restoredText->get(), - sizeof(char32_t) * ucharCount); - pos += static_cast<int>(ucharCount); - len += static_cast<int>(ucharCount); - refreshLine(pi); - killRing.lastAction = KillRing::actionYank; - killRing.lastYankSize = ucharCount; - if (truncated) { - beep(); - } - } else { - beep(); - } - } - break; - - case META + 'y': // meta-Y, "yank-pop", rotate popped text - case META + 'Y': - if (killRing.lastAction == KillRing::actionYank) { - historyRecallMostRecent = false; - Utf32String* restoredText = killRing.yankPop(); - if (restoredText) { - bool truncated = false; - size_t ucharCount = restoredText->length(); - if (ucharCount > - static_cast<size_t>(killRing.lastYankSize + buflen - len)) { - ucharCount = killRing.lastYankSize + buflen - len; - truncated = true; - } - if (ucharCount > killRing.lastYankSize) { - memmove(buf32 + pos + ucharCount - killRing.lastYankSize, - buf32 + pos, sizeof(char32_t) * (len - pos + 1)); - memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(), - sizeof(char32_t) * ucharCount); - } else { - memmove(buf32 + pos - killRing.lastYankSize, restoredText->get(), - sizeof(char32_t) * ucharCount); - memmove(buf32 + pos + ucharCount - killRing.lastYankSize, - buf32 + pos, sizeof(char32_t) * (len - pos + 1)); - } - pos += static_cast<int>(ucharCount - killRing.lastYankSize); - len += static_cast<int>(ucharCount - killRing.lastYankSize); - killRing.lastYankSize = ucharCount; - refreshLine(pi); - if (truncated) { - beep(); - } - break; - } - } - beep(); - break; - -#ifndef _WIN32 - case ctrlChar('Z'): // ctrl-Z, job control - disableRawMode(); // Returning to Linux (whatever) shell, leave raw - // mode - raise(SIGSTOP); // Break out in mid-line - enableRawMode(); // Back from Linux shell, re-enter raw mode - if (!pi.write()) break; // Redraw prompt - refreshLine(pi); // Refresh the line - break; -#endif - - // DEL, delete the character under the cursor - case 127: - case DELETE_KEY: - killRing.lastAction = KillRing::actionOther; - if (len > 0 && pos < len) { - historyRecallMostRecent = false; - memmove(buf32 + pos, buf32 + pos + 1, sizeof(char32_t) * (len - pos)); - --len; - refreshLine(pi); - } - break; - - case META + '<': // meta-<, beginning of history - case PAGE_UP_KEY: // Page Up, beginning of history - case META + '>': // meta->, end of history - case PAGE_DOWN_KEY: // Page Down, end of history - killRing.lastAction = KillRing::actionOther; - // if not already recalling, add the current line to the history list so - // we don't - // have to special case it - if (historyIndex == historyLen - 1) { - free(history[historyLen - 1]); - size_t tempBufferSize = sizeof(char32_t) * len + 1; - unique_ptr<char[]> tempBuffer(new char[tempBufferSize]); - copyString32to8(tempBuffer.get(), tempBufferSize, buf32); - history[historyLen - 1] = strdup8(tempBuffer.get()); - } - if (historyLen > 1) { - historyIndex = - (c == META + '<' || c == PAGE_UP_KEY) ? 0 : historyLen - 1; - historyPreviousIndex = -2; - historyRecallMostRecent = true; - size_t ucharCount = 0; - copyString8to32(buf32, buflen, ucharCount, history[historyIndex]); - len = pos = static_cast<int>(ucharCount); - refreshLine(pi); - } - break; - - // not one of our special characters, maybe insert it in the buffer - default: - killRing.lastAction = KillRing::actionOther; - historyRecallMostRecent = false; - if (c & (META | CTRL)) { // beep on unknown Ctrl and/or Meta keys - beep(); - break; - } - if (len < buflen) { - if (isControlChar(c)) { // don't insert control characters - beep(); - break; - } - if (len == pos) { // at end of buffer - buf32[pos] = c; - ++pos; - ++len; - buf32[len] = '\0'; - int inputLen = calculateColumnPosition(buf32, len); - if (pi.promptIndentation + inputLen < pi.promptScreenColumns) { - if (inputLen > pi.promptPreviousInputLen) - pi.promptPreviousInputLen = inputLen; - /* Avoid a full update of the line in the - * trivial case. */ - if (write32(1, reinterpret_cast<char32_t*>(&c), 1) == -1) - return -1; - } else { - refreshLine(pi); - } - } else { // not at end of buffer, have to move characters to our - // right - memmove(buf32 + pos + 1, buf32 + pos, - sizeof(char32_t) * (len - pos)); - buf32[pos] = c; - ++len; - ++pos; - buf32[len] = '\0'; - refreshLine(pi); - } - } else { - beep(); // buffer is full, beep on new characters - } - break; - } - } - return len; -} - -static string preloadedBufferContents; // used with linenoisePreloadBuffer -static string preloadErrorMessage; - -/** - * linenoisePreloadBuffer provides text to be inserted into the command buffer - * - * the provided text will be processed to be usable and will be used to preload - * the input buffer on the next call to linenoise() - * - * @param preloadText text to begin with on the next call to linenoise() - */ -void linenoisePreloadBuffer(const char* preloadText) { - if (!preloadText) { - return; - } - int bufferSize = static_cast<int>(strlen(preloadText) + 1); - unique_ptr<char[]> tempBuffer(new char[bufferSize]); - strncpy(&tempBuffer[0], preloadText, bufferSize); - - // remove characters that won't display correctly - char* pIn = &tempBuffer[0]; - char* pOut = pIn; - bool controlsStripped = false; - bool whitespaceSeen = false; - while (*pIn) { - unsigned char c = - *pIn++; // we need unsigned so chars 0x80 and above are allowed - if ('\r' == c) { // silently skip CR - continue; - } - if ('\n' == c || '\t' == c) { // note newline or tab - whitespaceSeen = true; - continue; - } - if (isControlChar( - c)) { // remove other control characters, flag for message - controlsStripped = true; - *pOut++ = ' '; - continue; - } - if (whitespaceSeen) { // convert whitespace to a single space - *pOut++ = ' '; - whitespaceSeen = false; - } - *pOut++ = c; - } - *pOut = 0; - int processedLength = static_cast<int>(pOut - tempBuffer.get()); - bool lineTruncated = false; - if (processedLength > (LINENOISE_MAX_LINE - 1)) { - lineTruncated = true; - tempBuffer[LINENOISE_MAX_LINE - 1] = 0; - } - preloadedBufferContents = tempBuffer.get(); - if (controlsStripped) { - preloadErrorMessage += - " [Edited line: control characters were converted to spaces]\n"; - } - if (lineTruncated) { - preloadErrorMessage += " [Edited line: the line length was reduced from "; - char buf[128]; - snprintf(buf, sizeof(buf), "%d to %d]\n", processedLength, - (LINENOISE_MAX_LINE - 1)); - preloadErrorMessage += buf; - } -} - -/** - * linenoise is a readline replacement. - * - * call it with a prompt to display and it will return a line of input from the - * user - * - * @param prompt text of prompt to display to the user - * @return the returned string belongs to the caller on return and must be - * freed to prevent - * memory leaks - */ -char* linenoise(const char* prompt) { -#ifndef _WIN32 - gotResize = false; -#endif - if (isatty(STDIN_FILENO)) { // input is from a terminal - char32_t buf32[LINENOISE_MAX_LINE]; - char charWidths[LINENOISE_MAX_LINE]; - if (!preloadErrorMessage.empty()) { - printf("%s", preloadErrorMessage.c_str()); - fflush(stdout); - preloadErrorMessage.clear(); - } - PromptInfo pi(prompt, getScreenColumns()); - if (isUnsupportedTerm()) { - if (!pi.write()) return 0; - fflush(stdout); - if (preloadedBufferContents.empty()) { - unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]); - if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) { - return NULL; - } - size_t len = strlen(buf8.get()); - while (len && (buf8[len - 1] == '\n' || buf8[len - 1] == '\r')) { - --len; - buf8[len] = '\0'; - } - return strdup(buf8.get()); // caller must free buffer - } else { - char* buf8 = strdup(preloadedBufferContents.c_str()); - preloadedBufferContents.clear(); - return buf8; // caller must free buffer - } - } else { - if (enableRawMode() == -1) { - return NULL; - } - InputBuffer ib(buf32, charWidths, LINENOISE_MAX_LINE); - if (!preloadedBufferContents.empty()) { - ib.preloadBuffer(preloadedBufferContents.c_str()); - preloadedBufferContents.clear(); - } - int count = ib.getInputLine(pi); - disableRawMode(); - printf("\n"); - if (count == -1) { - return NULL; - } - size_t bufferSize = sizeof(char32_t) * ib.length() + 1; - unique_ptr<char[]> buf8(new char[bufferSize]); - copyString32to8(buf8.get(), bufferSize, buf32); - return strdup(buf8.get()); // caller must free buffer - } - } else { // input not from a terminal, we should work with piped input, i.e. - // redirected stdin - unique_ptr<char[]> buf8(new char[LINENOISE_MAX_LINE]); - if (fgets(buf8.get(), LINENOISE_MAX_LINE, stdin) == NULL) { - return NULL; - } - - // if fgets() gave us the newline, remove it - int count = static_cast<int>(strlen(buf8.get())); - if (count > 0 && buf8[count - 1] == '\n') { - --count; - buf8[count] = '\0'; - } - return strdup(buf8.get()); // caller must free buffer - } -} - -/* Register a callback function to be called for tab-completion. */ -void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn) { - completionCallback = fn; -} - -void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str) { - lc->completionStrings.push_back(Utf32String(str)); -} - -int linenoiseHistoryAdd(const char* line) { - if (historyMaxLen == 0) { - return 0; - } - if (history == NULL) { - history = - reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * historyMaxLen)); - if (history == NULL) { - return 0; - } - memset(history, 0, (sizeof(char*) * historyMaxLen)); - } - char8_t* linecopy = strdup8(line); - if (!linecopy) { - return 0; - } - - // convert newlines in multi-line code to spaces before storing - char8_t* p = linecopy; - while (*p) { - if (*p == '\n') { - *p = ' '; - } - ++p; - } - - // prevent duplicate history entries - if (historyLen > 0 && history[historyLen - 1] != nullptr && - strcmp(reinterpret_cast<char const*>(history[historyLen - 1]), - reinterpret_cast<char const*>(linecopy)) == 0) { - free(linecopy); - return 0; - } - - if (historyLen == historyMaxLen) { - free(history[0]); - memmove(history, history + 1, sizeof(char*) * (historyMaxLen - 1)); - --historyLen; - if (--historyPreviousIndex < -1) { - historyPreviousIndex = -2; - } - } - - history[historyLen] = linecopy; - ++historyLen; - return 1; -} - -int linenoiseHistorySetMaxLen(int len) { - if (len < 1) { - return 0; - } - if (history) { - int tocopy = historyLen; - char8_t** newHistory = - reinterpret_cast<char8_t**>(malloc(sizeof(char8_t*) * len)); - if (newHistory == NULL) { - return 0; - } - if (len < tocopy) { - tocopy = len; - } - memcpy(newHistory, history + historyMaxLen - tocopy, - sizeof(char8_t*) * tocopy); - free(history); - history = newHistory; - } - historyMaxLen = len; - if (historyLen > historyMaxLen) { - historyLen = historyMaxLen; - } - return 1; -} - -/* Fetch a line of the history by (zero-based) index. If the requested - * line does not exist, NULL is returned. The return value is a heap-allocated - * copy of the line, and the caller is responsible for de-allocating it. */ -char* linenoiseHistoryLine(int index) { - if (index < 0 || index >= historyLen) return NULL; - - return strdup(reinterpret_cast<char const*>(history[index])); -} - -/* Save the history in the specified file. On success 0 is returned - * otherwise -1 is returned. */ -int linenoiseHistorySave(const char* filename) { -#if _WIN32 - FILE* fp = fopen(filename, "wt"); -#else - int fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); - - if (fd < 0) { - return -1; - } - - FILE* fp = fdopen(fd, "wt"); -#endif - - if (fp == NULL) { - return -1; - } - - for (int j = 0; j < historyLen; ++j) { - if (history[j][0] != '\0') { - fprintf(fp, "%s\n", history[j]); - } - } - - fclose(fp); - - return 0; -} - -/* Load the history from the specified file. If the file does not exist - * zero is returned and no operation is performed. - * - * If the file exists and the operation succeeded 0 is returned, otherwise - * on error -1 is returned. */ -int linenoiseHistoryLoad(const char* filename) { - FILE* fp = fopen(filename, "rt"); - if (fp == NULL) { - return -1; - } - - char buf[LINENOISE_MAX_LINE]; - while (fgets(buf, LINENOISE_MAX_LINE, fp) != NULL) { - char* p = strchr(buf, '\r'); - if (!p) { - p = strchr(buf, '\n'); - } - if (p) { - *p = '\0'; - } - if (p != buf) { - linenoiseHistoryAdd(buf); - } - } - fclose(fp); - return 0; -} - -/* Set if to use or not the multi line mode. */ -/* note that this is a stub only, as linenoise-ng always multi-line */ -void linenoiseSetMultiLine(int) {} - -/* This special mode is used by linenoise in order to print scan codes - * on screen for debugging / development purposes. It is implemented - * by the linenoise_example program using the --keycodes option. */ -void linenoisePrintKeyCodes(void) { - char quit[4]; - - printf( - "Linenoise key codes debugging mode.\n" - "Press keys to see scan codes. Type 'quit' at any time to exit.\n"); - if (enableRawMode() == -1) return; - memset(quit, ' ', 4); - while (1) { - char c; - int nread; - -#if _WIN32 - nread = _read(STDIN_FILENO, &c, 1); -#else - nread = read(STDIN_FILENO, &c, 1); -#endif - if (nread <= 0) continue; - memmove(quit, quit + 1, sizeof(quit) - 1); /* shift string to left. */ - quit[sizeof(quit) - 1] = c; /* Insert current char on the right. */ - if (memcmp(quit, "quit", sizeof(quit)) == 0) break; - - printf("'%c' %02x (%d) (type quit to exit)\n", isprint(c) ? c : '?', (int)c, - (int)c); - printf("\r"); /* Go left edge manually, we are in raw mode. */ - fflush(stdout); - } - disableRawMode(); -} - -#ifndef _WIN32 -static void WindowSizeChanged(int) { - // do nothing here but setting this flag - gotResize = true; -} -#endif - -int linenoiseInstallWindowChangeHandler(void) { -#ifndef _WIN32 - struct sigaction sa; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - sa.sa_handler = &WindowSizeChanged; - - if (sigaction(SIGWINCH, &sa, nullptr) == -1) { - return errno; - } -#endif - return 0; -} - -int linenoiseKeyType(void) { - return keyType; -} diff --git a/src/linenoise/linenoise.h b/src/linenoise/linenoise.h deleted file mode 100644 index 3a8eb9f7e..000000000 --- a/src/linenoise/linenoise.h +++ /dev/null @@ -1,73 +0,0 @@ -/* linenoise.h -- guerrilla line editing library against the idea that a - * line editing lib needs to be 20,000 lines of C code. - * - * See linenoise.c for more information. - * - * Copyright (c) 2010, Salvatore Sanfilippo <antirez at gmail dot com> - * Copyright (c) 2010, Pieter Noordhuis <pcnoordhuis at gmail dot com> - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef __LINENOISE_H -#define __LINENOISE_H - -#define LINENOISE_VERSION "1.0.0" -#define LINENOISE_VERSION_MAJOR 1 -#define LINENOISE_VERSION_MINOR 1 - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct linenoiseCompletions linenoiseCompletions; - -typedef void(linenoiseCompletionCallback)(const char*, linenoiseCompletions*); -void linenoiseSetCompletionCallback(linenoiseCompletionCallback* fn); -void linenoiseAddCompletion(linenoiseCompletions* lc, const char* str); - -char* linenoise(const char* prompt); -void linenoisePreloadBuffer(const char* preloadText); -int linenoiseHistoryAdd(const char* line); -int linenoiseHistorySetMaxLen(int len); -char* linenoiseHistoryLine(int index); -int linenoiseHistorySave(const char* filename); -int linenoiseHistoryLoad(const char* filename); -void linenoiseHistoryFree(void); -void linenoiseClearScreen(void); -void linenoiseSetMultiLine(int ml); -void linenoisePrintKeyCodes(void); -/* the following are extensions to the original linenoise API */ -int linenoiseInstallWindowChangeHandler(void); -/* returns type of key pressed: 1 = CTRL-C, 2 = CTRL-D, 0 = other */ -int linenoiseKeyType(void); - -#ifdef __cplusplus -} -#endif - -#endif /* __LINENOISE_H */ diff --git a/src/linenoise/wcwidth.cpp b/src/linenoise/wcwidth.cpp deleted file mode 100644 index deec0ba6b..000000000 --- a/src/linenoise/wcwidth.cpp +++ /dev/null @@ -1,315 +0,0 @@ -/* - * This is an implementation of wcwidth() and wcswidth() (defined in - * IEEE Std 1002.1-2001) for Unicode. - * - * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html - * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html - * - * In fixed-width output devices, Latin characters all occupy a single - * "cell" position of equal width, whereas ideographic CJK characters - * occupy two such cells. Interoperability between terminal-line - * applications and (teletype-style) character terminals using the - * UTF-8 encoding requires agreement on which character should advance - * the cursor by how many cell positions. No established formal - * standards exist at present on which Unicode character shall occupy - * how many cell positions on character terminals. These routines are - * a first attempt of defining such behavior based on simple rules - * applied to data provided by the Unicode Consortium. - * - * For some graphical characters, the Unicode standard explicitly - * defines a character-cell width via the definition of the East Asian - * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes. - * In all these cases, there is no ambiguity about which width a - * terminal shall use. For characters in the East Asian Ambiguous (A) - * class, the width choice depends purely on a preference of backward - * compatibility with either historic CJK or Western practice. - * Choosing single-width for these characters is easy to justify as - * the appropriate long-term solution, as the CJK practice of - * displaying these characters as double-width comes from historic - * implementation simplicity (8-bit encoded characters were displayed - * single-width and 16-bit ones double-width, even for Greek, - * Cyrillic, etc.) and not any typographic considerations. - * - * Much less clear is the choice of width for the Not East Asian - * (Neutral) class. Existing practice does not dictate a width for any - * of these characters. It would nevertheless make sense - * typographically to allocate two character cells to characters such - * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be - * represented adequately with a single-width glyph. The following - * routines at present merely assign a single-cell width to all - * neutral characters, in the interest of simplicity. This is not - * entirely satisfactory and should be reconsidered before - * establishing a formal standard in this area. At the moment, the - * decision which Not East Asian (Neutral) characters should be - * represented by double-width glyphs cannot yet be answered by - * applying a simple rule from the Unicode database content. Setting - * up a proper standard for the behavior of UTF-8 character terminals - * will require a careful analysis not only of each Unicode character, - * but also of each presentation form, something the author of these - * routines has avoided to do so far. - * - * http://www.unicode.org/unicode/reports/tr11/ - * - * Markus Kuhn -- 2007-05-26 (Unicode 5.0) - * - * Permission to use, copy, modify, and distribute this software - * for any purpose and without fee is hereby granted. The author - * disclaims all warranties with regard to this software. - * - * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c - */ - -#include <wchar.h> -#include <string> -#include <memory> - -namespace linenoise_ng { - -struct interval { - char32_t first; - char32_t last; -}; - -/* auxiliary function for binary search in interval table */ -static int bisearch(char32_t ucs, const struct interval *table, int max) { - int min = 0; - int mid; - - if (ucs < table[0].first || ucs > table[max].last) - return 0; - while (max >= min) { - mid = (min + max) / 2; - if (ucs > table[mid].last) - min = mid + 1; - else if (ucs < table[mid].first) - max = mid - 1; - else - return 1; - } - - return 0; -} - - -/* The following two functions define the column width of an ISO 10646 - * character as follows: - * - * - The null character (U+0000) has a column width of 0. - * - * - Other C0/C1 control characters and DEL will lead to a return - * value of -1. - * - * - Non-spacing and enclosing combining characters (general - * category code Mn or Me in the Unicode database) have a - * column width of 0. - * - * - SOFT HYPHEN (U+00AD) has a column width of 1. - * - * - Other format characters (general category code Cf in the Unicode - * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. - * - * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) - * have a column width of 0. - * - * - Spacing characters in the East Asian Wide (W) or East Asian - * Full-width (F) category as defined in Unicode Technical - * Report #11 have a column width of 2. - * - * - All remaining characters (including all printable - * ISO 8859-1 and WGL4 characters, Unicode control characters, - * etc.) have a column width of 1. - * - * This implementation assumes that wchar_t characters are encoded - * in ISO 10646. - */ - -int mk_wcwidth(char32_t ucs) -{ - /* sorted list of non-overlapping intervals of non-spacing characters */ - /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ - static const struct interval combining[] = { - { 0x0300, 0x036F }, { 0x0483, 0x0486 }, { 0x0488, 0x0489 }, - { 0x0591, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, - { 0x05C4, 0x05C5 }, { 0x05C7, 0x05C7 }, { 0x0600, 0x0603 }, - { 0x0610, 0x0615 }, { 0x064B, 0x065E }, { 0x0670, 0x0670 }, - { 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, - { 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A }, - { 0x07A6, 0x07B0 }, { 0x07EB, 0x07F3 }, { 0x0901, 0x0902 }, - { 0x093C, 0x093C }, { 0x0941, 0x0948 }, { 0x094D, 0x094D }, - { 0x0951, 0x0954 }, { 0x0962, 0x0963 }, { 0x0981, 0x0981 }, - { 0x09BC, 0x09BC }, { 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, - { 0x09E2, 0x09E3 }, { 0x0A01, 0x0A02 }, { 0x0A3C, 0x0A3C }, - { 0x0A41, 0x0A42 }, { 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, - { 0x0A70, 0x0A71 }, { 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC }, - { 0x0AC1, 0x0AC5 }, { 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD }, - { 0x0AE2, 0x0AE3 }, { 0x0B01, 0x0B01 }, { 0x0B3C, 0x0B3C }, - { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, { 0x0B4D, 0x0B4D }, - { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, { 0x0BC0, 0x0BC0 }, - { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, { 0x0C46, 0x0C48 }, - { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, { 0x0CBC, 0x0CBC }, - { 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, - { 0x0CE2, 0x0CE3 }, { 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D }, - { 0x0DCA, 0x0DCA }, { 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 }, - { 0x0E31, 0x0E31 }, { 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E }, - { 0x0EB1, 0x0EB1 }, { 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC }, - { 0x0EC8, 0x0ECD }, { 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 }, - { 0x0F37, 0x0F37 }, { 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E }, - { 0x0F80, 0x0F84 }, { 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 }, - { 0x0F99, 0x0FBC }, { 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 }, - { 0x1032, 0x1032 }, { 0x1036, 0x1037 }, { 0x1039, 0x1039 }, - { 0x1058, 0x1059 }, { 0x1160, 0x11FF }, { 0x135F, 0x135F }, - { 0x1712, 0x1714 }, { 0x1732, 0x1734 }, { 0x1752, 0x1753 }, - { 0x1772, 0x1773 }, { 0x17B4, 0x17B5 }, { 0x17B7, 0x17BD }, - { 0x17C6, 0x17C6 }, { 0x17C9, 0x17D3 }, { 0x17DD, 0x17DD }, - { 0x180B, 0x180D }, { 0x18A9, 0x18A9 }, { 0x1920, 0x1922 }, - { 0x1927, 0x1928 }, { 0x1932, 0x1932 }, { 0x1939, 0x193B }, - { 0x1A17, 0x1A18 }, { 0x1B00, 0x1B03 }, { 0x1B34, 0x1B34 }, - { 0x1B36, 0x1B3A }, { 0x1B3C, 0x1B3C }, { 0x1B42, 0x1B42 }, - { 0x1B6B, 0x1B73 }, { 0x1DC0, 0x1DCA }, { 0x1DFE, 0x1DFF }, - { 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x2060, 0x2063 }, - { 0x206A, 0x206F }, { 0x20D0, 0x20EF }, { 0x302A, 0x302F }, - { 0x3099, 0x309A }, { 0xA806, 0xA806 }, { 0xA80B, 0xA80B }, - { 0xA825, 0xA826 }, { 0xFB1E, 0xFB1E }, { 0xFE00, 0xFE0F }, - { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, { 0xFFF9, 0xFFFB }, - { 0x10A01, 0x10A03 }, { 0x10A05, 0x10A06 }, { 0x10A0C, 0x10A0F }, - { 0x10A38, 0x10A3A }, { 0x10A3F, 0x10A3F }, { 0x1D167, 0x1D169 }, - { 0x1D173, 0x1D182 }, { 0x1D185, 0x1D18B }, { 0x1D1AA, 0x1D1AD }, - { 0x1D242, 0x1D244 }, { 0xE0001, 0xE0001 }, { 0xE0020, 0xE007F }, - { 0xE0100, 0xE01EF } - }; - - /* test for 8-bit control characters */ - if (ucs == 0) - return 0; - if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0)) - return -1; - - /* binary search in table of non-spacing characters */ - if (bisearch(ucs, combining, - sizeof(combining) / sizeof(struct interval) - 1)) - return 0; - - /* if we arrive here, ucs is not a combining or C0/C1 control character */ - - return 1 + - (ucs >= 0x1100 && - (ucs <= 0x115f || /* Hangul Jamo init. consonants */ - ucs == 0x2329 || ucs == 0x232a || - (ucs >= 0x2e80 && ucs <= 0xa4cf && - ucs != 0x303f) || /* CJK ... Yi */ - (ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ - (ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */ - (ucs >= 0xfe10 && ucs <= 0xfe19) || /* Vertical forms */ - (ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ - (ucs >= 0xff00 && ucs <= 0xff60) || /* Fullwidth Forms */ - (ucs >= 0xffe0 && ucs <= 0xffe6) || - (ucs >= 0x20000 && ucs <= 0x2fffd) || - (ucs >= 0x30000 && ucs <= 0x3fffd))); -} - - -int mk_wcswidth(const char32_t* pwcs, size_t n) -{ - int w, width = 0; - - for (;*pwcs && n-- > 0; pwcs++) - if ((w = mk_wcwidth(*pwcs)) < 0) - return -1; - else - width += w; - - return width; -} - - -/* - * The following functions are the same as mk_wcwidth() and - * mk_wcswidth(), except that spacing characters in the East Asian - * Ambiguous (A) category as defined in Unicode Technical Report #11 - * have a column width of 2. This variant might be useful for users of - * CJK legacy encodings who want to migrate to UCS without changing - * the traditional terminal character-width behaviour. It is not - * otherwise recommended for general use. - */ -int mk_wcwidth_cjk(wchar_t ucs) -{ - /* sorted list of non-overlapping intervals of East Asian Ambiguous - * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */ - static const struct interval ambiguous[] = { - { 0x00A1, 0x00A1 }, { 0x00A4, 0x00A4 }, { 0x00A7, 0x00A8 }, - { 0x00AA, 0x00AA }, { 0x00AE, 0x00AE }, { 0x00B0, 0x00B4 }, - { 0x00B6, 0x00BA }, { 0x00BC, 0x00BF }, { 0x00C6, 0x00C6 }, - { 0x00D0, 0x00D0 }, { 0x00D7, 0x00D8 }, { 0x00DE, 0x00E1 }, - { 0x00E6, 0x00E6 }, { 0x00E8, 0x00EA }, { 0x00EC, 0x00ED }, - { 0x00F0, 0x00F0 }, { 0x00F2, 0x00F3 }, { 0x00F7, 0x00FA }, - { 0x00FC, 0x00FC }, { 0x00FE, 0x00FE }, { 0x0101, 0x0101 }, - { 0x0111, 0x0111 }, { 0x0113, 0x0113 }, { 0x011B, 0x011B }, - { 0x0126, 0x0127 }, { 0x012B, 0x012B }, { 0x0131, 0x0133 }, - { 0x0138, 0x0138 }, { 0x013F, 0x0142 }, { 0x0144, 0x0144 }, - { 0x0148, 0x014B }, { 0x014D, 0x014D }, { 0x0152, 0x0153 }, - { 0x0166, 0x0167 }, { 0x016B, 0x016B }, { 0x01CE, 0x01CE }, - { 0x01D0, 0x01D0 }, { 0x01D2, 0x01D2 }, { 0x01D4, 0x01D4 }, - { 0x01D6, 0x01D6 }, { 0x01D8, 0x01D8 }, { 0x01DA, 0x01DA }, - { 0x01DC, 0x01DC }, { 0x0251, 0x0251 }, { 0x0261, 0x0261 }, - { 0x02C4, 0x02C4 }, { 0x02C7, 0x02C7 }, { 0x02C9, 0x02CB }, - { 0x02CD, 0x02CD }, { 0x02D0, 0x02D0 }, { 0x02D8, 0x02DB }, - { 0x02DD, 0x02DD }, { 0x02DF, 0x02DF }, { 0x0391, 0x03A1 }, - { 0x03A3, 0x03A9 }, { 0x03B1, 0x03C1 }, { 0x03C3, 0x03C9 }, - { 0x0401, 0x0401 }, { 0x0410, 0x044F }, { 0x0451, 0x0451 }, - { 0x2010, 0x2010 }, { 0x2013, 0x2016 }, { 0x2018, 0x2019 }, - { 0x201C, 0x201D }, { 0x2020, 0x2022 }, { 0x2024, 0x2027 }, - { 0x2030, 0x2030 }, { 0x2032, 0x2033 }, { 0x2035, 0x2035 }, - { 0x203B, 0x203B }, { 0x203E, 0x203E }, { 0x2074, 0x2074 }, - { 0x207F, 0x207F }, { 0x2081, 0x2084 }, { 0x20AC, 0x20AC }, - { 0x2103, 0x2103 }, { 0x2105, 0x2105 }, { 0x2109, 0x2109 }, - { 0x2113, 0x2113 }, { 0x2116, 0x2116 }, { 0x2121, 0x2122 }, - { 0x2126, 0x2126 }, { 0x212B, 0x212B }, { 0x2153, 0x2154 }, - { 0x215B, 0x215E }, { 0x2160, 0x216B }, { 0x2170, 0x2179 }, - { 0x2190, 0x2199 }, { 0x21B8, 0x21B9 }, { 0x21D2, 0x21D2 }, - { 0x21D4, 0x21D4 }, { 0x21E7, 0x21E7 }, { 0x2200, 0x2200 }, - { 0x2202, 0x2203 }, { 0x2207, 0x2208 }, { 0x220B, 0x220B }, - { 0x220F, 0x220F }, { 0x2211, 0x2211 }, { 0x2215, 0x2215 }, - { 0x221A, 0x221A }, { 0x221D, 0x2220 }, { 0x2223, 0x2223 }, - { 0x2225, 0x2225 }, { 0x2227, 0x222C }, { 0x222E, 0x222E }, - { 0x2234, 0x2237 }, { 0x223C, 0x223D }, { 0x2248, 0x2248 }, - { 0x224C, 0x224C }, { 0x2252, 0x2252 }, { 0x2260, 0x2261 }, - { 0x2264, 0x2267 }, { 0x226A, 0x226B }, { 0x226E, 0x226F }, - { 0x2282, 0x2283 }, { 0x2286, 0x2287 }, { 0x2295, 0x2295 }, - { 0x2299, 0x2299 }, { 0x22A5, 0x22A5 }, { 0x22BF, 0x22BF }, - { 0x2312, 0x2312 }, { 0x2460, 0x24E9 }, { 0x24EB, 0x254B }, - { 0x2550, 0x2573 }, { 0x2580, 0x258F }, { 0x2592, 0x2595 }, - { 0x25A0, 0x25A1 }, { 0x25A3, 0x25A9 }, { 0x25B2, 0x25B3 }, - { 0x25B6, 0x25B7 }, { 0x25BC, 0x25BD }, { 0x25C0, 0x25C1 }, - { 0x25C6, 0x25C8 }, { 0x25CB, 0x25CB }, { 0x25CE, 0x25D1 }, - { 0x25E2, 0x25E5 }, { 0x25EF, 0x25EF }, { 0x2605, 0x2606 }, - { 0x2609, 0x2609 }, { 0x260E, 0x260F }, { 0x2614, 0x2615 }, - { 0x261C, 0x261C }, { 0x261E, 0x261E }, { 0x2640, 0x2640 }, - { 0x2642, 0x2642 }, { 0x2660, 0x2661 }, { 0x2663, 0x2665 }, - { 0x2667, 0x266A }, { 0x266C, 0x266D }, { 0x266F, 0x266F }, - { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF }, - { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD } - }; - - /* binary search in table of non-spacing characters */ - if (bisearch(ucs, ambiguous, - sizeof(ambiguous) / sizeof(struct interval) - 1)) - return 2; - - return mk_wcwidth(ucs); -} - - -int mk_wcswidth_cjk(const wchar_t *pwcs, size_t n) -{ - int w, width = 0; - - for (;*pwcs && n-- > 0; pwcs++) - if ((w = mk_wcwidth_cjk(*pwcs)) < 0) - return -1; - else - width += w; - - return width; -} - -} diff --git a/src/nix-build/local.mk b/src/nix-build/local.mk deleted file mode 100644 index a2d1c91df..000000000 --- a/src/nix-build/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-build - -nix-build_DIR := $(d) - -nix-build_SOURCES := $(d)/nix-build.cc - -nix-build_LIBS = libmain libexpr libstore libutil libformat - -$(eval $(call install-symlink, nix-build, $(bindir)/nix-shell)) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 34f1cba9d..33ad28704 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -16,6 +16,7 @@ #include "get-drvs.hh" #include "common-eval-args.hh" #include "attr-path.hh" +#include "legacy.hh" using namespace nix; using namespace std::string_literals; @@ -66,11 +67,8 @@ std::vector<string> shellwords(const string & s) return res; } -void mainWrapped(int argc, char * * argv) +static void _main(int argc, char * * argv) { - initNix(); - initGC(); - auto dryRun = false; auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); auto pure = false; @@ -276,19 +274,21 @@ void mainWrapped(int argc, char * * argv) exprs = {state->parseStdin()}; else for (auto i : left) { - auto absolute = i; - try { - absolute = canonPath(absPath(i), true); - } catch (Error e) {}; if (fromArgs) exprs.push_back(state->parseExprFromString(i, absPath("."))); - else if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error & e) {}; + if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) drvs.push_back(DrvInfo(*state, store, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + } } /* Evaluate them into derivations. */ @@ -305,6 +305,8 @@ void mainWrapped(int argc, char * * argv) } } + state->printStats(); + auto buildPaths = [&](const PathSet & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ @@ -415,17 +417,20 @@ void mainWrapped(int argc, char * * argv) "dontAddDisableDepTrack=1; " "[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" + "PATH=\"%4%:$PATH\"; " + "SHELL=%5%; " "set +e; " R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " - "unset NIX_INDENT_MAKE; " "shopt -u nullglob; " - "unset TZ; %4%" - "%5%", + "unset TZ; %6%" + "%7%", (Path) tmpDir, (pure ? "" : "p=$PATH; "), (pure ? "" : "PATH=$PATH:$p; unset p; "), + dirOf(shell), + shell, (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), envCommand)); @@ -499,9 +504,5 @@ void mainWrapped(int argc, char * * argv) } } -int main(int argc, char * * argv) -{ - return handleExceptions(argv[0], [&]() { - return mainWrapped(argc, argv); - }); -} +static RegisterLegacyCommand s1("nix-build", _main); +static RegisterLegacyCommand s2("nix-shell", _main); diff --git a/src/nix-channel/local.mk b/src/nix-channel/local.mk deleted file mode 100644 index c14e8c359..000000000 --- a/src/nix-channel/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-channel - -nix-channel_DIR := $(d) - -nix-channel_LIBS = libmain libformat libstore libutil - -nix-channel_SOURCES := $(d)/nix-channel.cc diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 55ebda438..06eb3d23b 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -1,9 +1,11 @@ #include "shared.hh" #include "globals.hh" #include "download.hh" +#include "store-api.hh" +#include "legacy.hh" + #include <fcntl.h> #include <regex> -#include "store-api.hh" #include <pwd.h> using namespace nix; @@ -84,10 +86,12 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - std::string effectiveUrl; + CachedDownloadRequest request(url); + request.ttl = 0; auto dl = getDownloader(); - auto filename = dl->downloadCached(store, url, false, "", Hash(), &effectiveUrl, 0); - url = chomp(std::move(effectiveUrl)); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -109,22 +113,11 @@ static void update(const StringSet & channelNames) } if (!unpacked) { - // The URL doesn't unpack directly, so let's try treating it like a full channel folder with files in it - // Check if the channel advertises a binary cache. - DownloadRequest request(url + "/binary-cache-url"); - try { - auto dlRes = dl->download(request); - extraAttrs = "binaryCacheURL = \"" + *dlRes.data + "\";"; - } catch (DownloadError & e) { - } - // Download the channel tarball. - auto fullURL = url + "/nixexprs.tar.xz"; try { - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; } catch (DownloadError & e) { - fullURL = url + "/nixexprs.tar.bz2"; - filename = dl->downloadCached(store, fullURL, false); + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; } chomp(filename); } @@ -157,11 +150,9 @@ static void update(const StringSet & channelNames) replaceSymlink(profile, channelLink); } -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = home + "/.nix-channels"; @@ -169,7 +160,7 @@ int main(int argc, char ** argv) // Figure out the name of the channels profile. ; - auto pw = getpwuid(getuid()); + auto pw = getpwuid(geteuid()); std::string name = pw ? pw->pw_name : getEnv("USER", ""); if (name.empty()) throw Error("cannot figure out user name"); @@ -255,5 +246,9 @@ int main(int argc, char ** argv) runProgram(settings.nixBinDir + "/nix-env", false, envArgs); break; } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/src/nix-collect-garbage/local.mk b/src/nix-collect-garbage/local.mk deleted file mode 100644 index 02d14cf62..000000000 --- a/src/nix-collect-garbage/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-collect-garbage - -nix-collect-garbage_DIR := $(d) - -nix-collect-garbage_SOURCES := $(d)/nix-collect-garbage.cc - -nix-collect-garbage_LIBS = libmain libstore libutil libformat diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 37fe22f48..d4060ac93 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -2,6 +2,7 @@ #include "profiles.hh" #include "shared.hh" #include "globals.hh" +#include "legacy.hh" #include <iostream> #include <cerrno> @@ -48,12 +49,10 @@ void removeOldGenerations(std::string dir) } } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - bool removeOld = false; - - return handleExceptions(argv[0], [&]() { - initNix(); + { + bool removeOld = false; GCOptions options; @@ -90,5 +89,9 @@ int main(int argc, char * * argv) PrintFreed freed(true, results); store->collectGarbage(options, results); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/src/nix-copy-closure/local.mk b/src/nix-copy-closure/local.mk deleted file mode 100644 index 5018ab975..000000000 --- a/src/nix-copy-closure/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-copy-closure - -nix-copy-closure_DIR := $(d) - -nix-copy-closure_LIBS = libmain libformat libstore libutil - -nix-copy-closure_SOURCES := $(d)/nix-copy-closure.cc diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index dfb1b8fc5..fdcde8b07 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,13 +1,12 @@ #include "shared.hh" #include "store-api.hh" +#include "legacy.hh" using namespace nix; -int main(int argc, char ** argv) +static int _main(int argc, char ** argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto gzip = false; auto toMode = true; auto includeOutputs = false; @@ -61,5 +60,9 @@ int main(int argc, char ** argv) from->computeFSClosure(storePaths2, closure, false, includeOutputs); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/src/nix-daemon/local.mk b/src/nix-daemon/local.mk deleted file mode 100644 index 5a4474465..000000000 --- a/src/nix-daemon/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -programs += nix-daemon - -nix-daemon_DIR := $(d) - -nix-daemon_SOURCES := $(d)/nix-daemon.cc - -nix-daemon_LIBS = libmain libstore libutil libformat - -nix-daemon_LDFLAGS = -pthread - -ifeq ($(OS), SunOS) - nix-daemon_LDFLAGS += -lsocket -endif diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 644fa6681..e88aaf636 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -9,6 +9,7 @@ #include "monitor-fd.hh" #include "derivations.hh" #include "finally.hh" +#include "legacy.hh" #include <algorithm> @@ -474,11 +475,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(); + Roots roots = store->findRoots(!trusted); logger->stopWork(); - to << roots.size(); + + size_t size = 0; for (auto & i : roots) - to << i.first << i.second; + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; + break; } @@ -557,14 +566,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ; else if (trusted || name == settings.buildTimeout.name - || name == "connect-timeout") + || name == "connect-timeout" + || (name == "builders" && value == "")) settings.set(name, value); else if (setSubstituters(settings.substituters)) ; else if (setSubstituters(settings.extraSubstituters)) ; else - debug("ignoring untrusted setting '%s'", name); + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { warn(e.what()); } @@ -708,7 +718,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); // FIXME: race if addToStore doesn't read source? - store.cast<Store>()->addToStore(info, *source, (RepairFlag) repair, + store->addToStore(info, *source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); logger->stopWork(); @@ -1057,11 +1067,9 @@ static void daemonLoop(char * * argv) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { auto stdio = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -1121,7 +1129,7 @@ int main(int argc, char * * argv) if (res == -1) throw SysError("splicing data from stdin to daemon socket"); else if (res == 0) - return; + return 0; } } } else { @@ -1130,5 +1138,9 @@ int main(int argc, char * * argv) } else { daemonLoop(argv); } - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/src/nix-env/local.mk b/src/nix-env/local.mk deleted file mode 100644 index e80719cd7..000000000 --- a/src/nix-env/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-env - -nix-env_DIR := $(d) - -nix-env_SOURCES := $(wildcard $(d)/*.cc) - -nix-env_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a43b103f6..87b2e43f0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -13,6 +13,7 @@ #include "json.hh" #include "value-to-json.hh" #include "xml-writer.hh" +#include "legacy.hh" #include <cerrno> #include <ctime> @@ -150,10 +151,8 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) if (stat(path.c_str(), &st) == -1) throw SysError(format("getting information about '%1%'") % path); - if (isNixExpr(path, st)) { + if (isNixExpr(path, st)) state.evalFile(path, v); - return; - } /* The path is a directory. Put the Nix expressions in the directory in a set, with the file name of each expression as @@ -161,13 +160,15 @@ static void loadSourceExpr(EvalState & state, const Path & path, Value & v) set flat, not nested, to make it easier for a user to have a ~/.nix-defexpr directory that includes some system-wide directory). */ - if (S_ISDIR(st.st_mode)) { + else if (S_ISDIR(st.st_mode)) { state.mkAttrs(v, 1024); state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); StringSet attrs; getAllExprs(state, path, attrs, v); v.attrs->sort(); } + + else throw Error("path '%s' is not a directory or a Nix expression", path); } @@ -859,7 +860,10 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems) for (auto & i : elems) { JSONObject pkgObj = topObj.object(i.attrPath); - pkgObj.attr("name", i.queryName()); + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); JSONObject metaObj = pkgObj.object("meta"); @@ -1025,10 +1029,14 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else if (printAttrPath) columns.push_back(i.attrPath); - if (xmlOutput) - attrs["name"] = i.queryName(); - else if (printName) + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { columns.push_back(i.queryName()); + } if (compareVersions) { /* Compare this element against the versions of the @@ -1311,12 +1319,9 @@ static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { Strings opFlags, opArgs; Operation op = 0; RepairFlag repair = NoRepair; @@ -1428,5 +1433,9 @@ int main(int argc, char * * argv) op(globals, opFlags, opArgs); globals.state->printStats(); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-env", _main); diff --git a/src/nix-instantiate/local.mk b/src/nix-instantiate/local.mk deleted file mode 100644 index 7d1bc5ec9..000000000 --- a/src/nix-instantiate/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-instantiate - -nix-instantiate_DIR := $(d) - -nix-instantiate_SOURCES := $(d)/nix-instantiate.cc - -nix-instantiate_LIBS = libexpr libmain libstore libutil libformat diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index eb6d34dd8..a736caa8f 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -9,6 +9,7 @@ #include "util.hh" #include "store-api.hh" #include "common-eval-args.hh" +#include "legacy.hh" #include <map> #include <iostream> @@ -83,12 +84,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { Strings files; bool readStdin = false; bool fromArgs = false; @@ -171,7 +169,7 @@ int main(int argc, char * * argv) if (p == "") throw Error(format("unable to find '%1%'") % i); std::cout << p << std::endl; } - return; + return 0; } if (readStdin) { @@ -190,5 +188,9 @@ int main(int argc, char * * argv) } state->printStats(); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/src/nix-prefetch-url/local.mk b/src/nix-prefetch-url/local.mk deleted file mode 100644 index 3e7735406..000000000 --- a/src/nix-prefetch-url/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -programs += nix-prefetch-url - -nix-prefetch-url_DIR := $(d) - -nix-prefetch-url_SOURCES := $(d)/nix-prefetch-url.cc - -nix-prefetch-url_LIBS = libmain libexpr libstore libutil libformat diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index a3b025723..f54706a8a 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -6,6 +6,9 @@ #include "eval-inline.hh" #include "common-eval-args.hh" #include "attr-path.hh" +#include "legacy.hh" +#include "finally.hh" +#include "progress-bar.hh" #include <iostream> @@ -44,12 +47,9 @@ string resolveMirrorUri(EvalState & state, string uri) } -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - initGC(); - + { HashType ht = htSHA256; std::vector<string> args; bool printPath = getEnv("PRINT_PATH") != ""; @@ -98,6 +98,11 @@ int main(int argc, char * * argv) if (args.size() > 2) throw UsageError("too many arguments"); + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + auto store = openStore(); auto state = std::make_unique<EvalState>(myArgs.searchPath, store); @@ -215,11 +220,17 @@ int main(int argc, char * * argv) assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); } + stopProgressBar(); + if (!printPath) printInfo(format("path is '%1%'") % storePath); std::cout << printHash16or32(hash) << std::endl; if (printPath) std::cout << storePath << std::endl; - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc new file mode 100644 index 000000000..670fbe227 --- /dev/null +++ b/src/nix-store/graphml.cc @@ -0,0 +1,90 @@ +#include "graphml.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" + +#include <iostream> + + +using std::cout; + +namespace nix { + + +static inline const string & xmlQuote(const string & s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + return string(p, p.find('-') + 1); +} + + +static string makeEdge(const string & src, const string & dst) +{ + return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", + xmlQuote(src), xmlQuote(dst)); +} + + +static string makeNode(const ValidPathInfo & info) +{ + return fmt( + " <node id=\"%1%\">\n" + " <data key=\"narSize\">%2%</data>\n" + " <data key=\"name\">%3%</data>\n" + " <data key=\"type\">%4%</data>\n" + " </node>\n", + info.path, + info.narSize, + symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); +} + + +void printGraphML(ref<Store> store, const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + std::pair<PathSet::iterator,bool> ret; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" + << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" + << " xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/graphml.xsd'>\n" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='name' for='node' attr.name='name' attr.type='string'/>" + << "<key id='type' for='node' attr.name='type' attr.type='string'/>" + << "<graph id='G' edgedefault='directed'>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (ret.second == false) continue; + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } + } + + } + + cout << "</graph>\n"; + cout << "</graphml>\n"; +} + + +} diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/graphml.hh index a6e7d4e28..b78df1e49 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/graphml.hh @@ -6,6 +6,6 @@ namespace nix { class Store; -void printXmlGraph(ref<Store> store, const PathSet & roots); +void printGraphML(ref<Store> store, const PathSet & roots); } diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk deleted file mode 100644 index ade0b233a..000000000 --- a/src/nix-store/local.mk +++ /dev/null @@ -1,9 +0,0 @@ -programs += nix-store - -nix-store_DIR := $(d) - -nix-store_SOURCES := $(wildcard $(d)/*.cc) - -nix-store_LIBS = libmain libstore libutil libformat - -nix-store_LDFLAGS = -lbz2 -pthread $(SODIUM_LIBS) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fe68f681a..0cbceb02f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -8,7 +8,8 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" -#include "xmlgraph.hh" +#include "graphml.hh" +#include "legacy.hh" #include <iostream> #include <algorithm> @@ -273,7 +274,7 @@ static void opQuery(Strings opFlags, Strings opArgs) enum QueryType { qDefault, qOutputs, qRequisites, qReferences, qReferrers , qReferrersClosure, qDeriver, qBinding, qHash, qSize - , qTree, qGraph, qXml, qResolve, qRoots }; + , qTree, qGraph, qGraphML, qResolve, qRoots }; QueryType query = qDefault; bool useOutput = false; bool includeOutputs = false; @@ -299,7 +300,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--size") query = qSize; else if (i == "--tree") query = qTree; else if (i == "--graph") query = qGraph; - else if (i == "--xml") query = qXml; + else if (i == "--graphml") query = qGraphML; else if (i == "--resolve") query = qResolve; else if (i == "--roots") query = qRoots; else if (i == "--use-output" || i == "-u") useOutput = true; @@ -403,13 +404,13 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } - case qXml: { + case qGraphML: { PathSet roots; for (auto & i : opArgs) { PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); roots.insert(paths.begin(), paths.end()); } - printXmlGraph(ref<Store>(store), roots); + printGraphML(ref<Store>(store), roots); break; } @@ -426,10 +427,11 @@ static void opQuery(Strings opFlags, Strings opArgs) maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); } - Roots roots = store->findRoots(); - for (auto & i : roots) - if (referrers.find(i.second) != referrers.end()) - cout << format("%1%\n") % i.first; + Roots roots = store->findRoots(false); + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) + for (auto & link : links) + cout << format("%1% -> %2%\n") % link % target; break; } @@ -484,11 +486,16 @@ static void opReadLog(Strings opFlags, Strings opArgs) static void opDumpDB(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); + if (!opArgs.empty()) { + for (auto & i : opArgs) + i = store->followLinksToStorePath(i); + for (auto & i : opArgs) + cout << store->makeValidityRegistration({i}, true, true); + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto & i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); + } } @@ -584,9 +591,14 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); if (printRoots) { - Roots roots = store->findRoots(); - for (auto & i : roots) - cout << i.first << " -> " << i.second << std::endl; + Roots roots = store->findRoots(false); + std::set<std::pair<Path, Path>> roots2; + // Transpose and sort the roots. + for (auto & [target, links] : roots) + for (auto & link : links) + roots2.emplace(link, target); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; } else { @@ -938,8 +950,16 @@ static void opServe(Strings opFlags, Strings opArgs) info.sigs = readStrings<StringSet>(in); in >> info.ca; - // FIXME: race if addToStore doesn't read source? - store->addToStore(info, in, NoRepair, NoCheckSigs); + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); out << 1; // indicate success @@ -993,11 +1013,9 @@ static void opVersion(Strings opFlags, Strings opArgs) /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ -int main(int argc, char * * argv) +static int _main(int argc, char * * argv) { - return handleExceptions(argv[0], [&]() { - initNix(); - + { Strings opFlags, opArgs; Operation op = 0; @@ -1084,5 +1102,9 @@ int main(int argc, char * * argv) store = openStore(); op(opFlags, opArgs); - }); + + return 0; + } } + +static RegisterLegacyCommand s1("nix-store", _main); diff --git a/src/nix-store/xmlgraph.cc b/src/nix-store/xmlgraph.cc deleted file mode 100644 index 0f7be7f7a..000000000 --- a/src/nix-store/xmlgraph.cc +++ /dev/null @@ -1,66 +0,0 @@ -#include "xmlgraph.hh" -#include "util.hh" -#include "store-api.hh" - -#include <iostream> - - -using std::cout; - -namespace nix { - - -static inline const string & xmlQuote(const string & s) -{ - // Luckily, store paths shouldn't contain any character that needs to be - // quoted. - return s; -} - - -static string makeEdge(const string & src, const string & dst) -{ - format f = format(" <edge src=\"%1%\" dst=\"%2%\"/>\n") - % xmlQuote(src) % xmlQuote(dst); - return f.str(); -} - - -static string makeNode(const string & id) -{ - format f = format(" <node name=\"%1%\"/>\n") % xmlQuote(id); - return f.str(); -} - - -void printXmlGraph(ref<Store> store, const PathSet & roots) -{ - PathSet workList(roots); - PathSet doneSet; - - cout << "<?xml version='1.0' encoding='utf-8'?>\n" - << "<nix>\n"; - - while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); - - if (doneSet.find(path) != doneSet.end()) continue; - doneSet.insert(path); - - cout << makeNode(path); - - for (auto & p : store->queryPathInfo(path)->references) { - if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); - } - } - - } - - cout << "</nix>\n"; -} - - -} diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index d0003790d..e86b96e3f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -8,7 +8,7 @@ using namespace nix; struct CmdAddToStore : MixDryRun, StoreCommand { Path path; - std::experimental::optional<std::string> namePart; + std::optional<std::string> namePart; CmdAddToStore() { diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 91711c8b4..12a9f9cd3 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -36,7 +36,7 @@ struct CmdCopy : StorePathsCommand .set(&checkSigs, NoCheckSigs); mkFlag() - .longName("substitute") + .longName("substitute-on-destination") .shortName('s') .description("whether to try substitutes on the destination store (only supported by SSH)") .set(&substitute, Substitute); @@ -69,12 +69,12 @@ struct CmdCopy : StorePathsCommand }, #ifdef ENABLE_S3 Example{ - "To populate the current folder build output to a S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1" + "To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" }, Example{ - "To populate the current folder build output to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com" + "To copy Hello to an S3-compatible binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" }, #endif }; diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc new file mode 100644 index 000000000..7b5444619 --- /dev/null +++ b/src/nix/doctor.cc @@ -0,0 +1,124 @@ +#include "command.hh" +#include "serve-protocol.hh" +#include "shared.hh" +#include "store-api.hh" +#include "worker-protocol.hh" + +using namespace nix; + +std::string formatProtocol(unsigned int proto) +{ + if (proto) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; +} + +struct CmdDoctor : StoreCommand +{ + bool success = true; + + std::string name() override + { + return "doctor"; + } + + std::string description() override + { + return "check your system for potential problems"; + } + + void run(ref<Store> store) override + { + std::cout << "Store uri: " << store->getUri() << std::endl; + std::cout << std::endl; + + auto type = getStoreType(); + + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); + } + success &= checkStoreProtocol(store->getProtocol()); + + if (!success) + throw Exit(2); + } + + bool checkNixInPath() + { + PathSet dirs; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) + dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + + if (dirs.size() != 1) { + std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkProfileRoots(ref<Store> store) + { + PathSet dirs; + + for (auto & dir : tokenizeString<Strings>(getEnv("PATH"), ":")) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); + + if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + + if (profileDir.find("/profiles/") == std::string::npos) + dirs.insert(dir); + } + } catch (SysError &) {} + } + + if (!dirs.empty()) { + std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; + std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; + std::cout << "garbage collected, resulting in broken symlinks." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkStoreProtocol(unsigned int storeProto) + { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::cout << "Warning: protocol version of this client does not match the store." << std::endl; + std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; + std::cout << "sync with the daemon." << std::endl; + std::cout << std::endl; + std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; + std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } +}; + +static RegisterCommand r1(make_ref<CmdDoctor>()); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index c9671f76d..a6169f118 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -55,7 +55,7 @@ struct CmdEdit : InstallableCommand int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); - } catch (std::invalid_argument e) { + } catch (std::invalid_argument & e) { throw Error("cannot parse line number '%s'", pos); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 64062fb97..af4105e28 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -9,13 +9,14 @@ struct CmdHash : Command { enum Mode { mFile, mPath }; Mode mode; - Base base = Base16; + Base base = SRI; bool truncate = false; - HashType ht = htSHA512; + HashType ht = htSHA256; std::vector<std::string> paths; CmdHash(Mode mode) : mode(mode) { + mkFlag(0, "sri", "print hash in SRI format", &base, SRI); mkFlag(0, "base64", "print hash in base-64", &base, Base64); mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); mkFlag(0, "base16", "print hash in base-16", &base, Base16); @@ -43,7 +44,7 @@ struct CmdHash : Command Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % - h.to_string(base, false); + h.to_string(base, base == SRI); } } }; @@ -54,7 +55,7 @@ static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); struct CmdToBase : Command { Base base; - HashType ht = htSHA512; + HashType ht = htUnknown; std::vector<std::string> args; CmdToBase(Base base) : base(base) @@ -70,26 +71,30 @@ struct CmdToBase : Command return base == Base16 ? "to-base16" : base == Base32 ? "to-base32" : - "to-base64"; + base == Base64 ? "to-base64" : + "to-sri"; } std::string description() override { - return fmt("convert a hash to base-%d representation", - base == Base16 ? 16 : - base == Base32 ? 32 : 64); + return fmt("convert a hash to %s representation", + base == Base16 ? "base-16" : + base == Base32 ? "base-32" : + base == Base64 ? "base-64" : + "SRI"); } void run() override { for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, false)); + std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); } }; static RegisterCommand r3(make_ref<CmdToBase>(Base16)); static RegisterCommand r4(make_ref<CmdToBase>(Base32)); static RegisterCommand r5(make_ref<CmdToBase>(Base64)); +static RegisterCommand r6(make_ref<CmdToBase>(SRI)); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 0be992b03..0c1ad3ab3 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -96,7 +96,7 @@ struct InstallableStorePath : Installable Buildables toBuildables() override { - return {{"", {{"out", storePath}}}}; + return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; } }; diff --git a/src/nix/local.mk b/src/nix/local.mk index f76da1944..c09efd1fc 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -2,10 +2,24 @@ programs += nix nix_DIR := $(d) -nix_SOURCES := $(wildcard $(d)/*.cc) $(wildcard src/linenoise/*.cpp) +nix_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard src/build-remote/*.cc) \ + $(wildcard src/nix-build/*.cc) \ + $(wildcard src/nix-channel/*.cc) \ + $(wildcard src/nix-collect-garbage/*.cc) \ + $(wildcard src/nix-copy-closure/*.cc) \ + $(wildcard src/nix-daemon/*.cc) \ + $(wildcard src/nix-env/*.cc) \ + $(wildcard src/nix-instantiate/*.cc) \ + $(wildcard src/nix-prefetch-url/*.cc) \ + $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil libformat +nix_LIBS = libexpr libmain libstore libutil -nix_LDFLAGS = -pthread +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -$(eval $(call install-symlink, nix, $(bindir)/nix-hash)) +$(foreach name, \ + nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ + $(eval $(call install-symlink, nix, $(bindir)/$(name)))) +$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) diff --git a/src/nix/ls.cc b/src/nix/ls.cc index e99622faf..d089be42f 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -148,7 +148,7 @@ struct CmdLsNar : Command, MixLs void run() override { - list(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true)))); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 9cd5d21c8..a80fd0ea6 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,23 +8,57 @@ #include "shared.hh" #include "store-api.hh" #include "progress-bar.hh" +#include "download.hh" #include "finally.hh" +#include <sys/types.h> +#include <sys/socket.h> +#include <ifaddrs.h> +#include <netdb.h> + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); namespace nix { +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) + return true; + } + } + + return false; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + bool useNet = true; + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { mkFlag() .longName("help") - .shortName('h') .description("show usage information") .handler([&]() { showHelpAndExit(); }); @@ -43,9 +77,20 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs }); mkFlag() + .longName("print-build-logs") + .shortName('L') + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + + mkFlag() .longName("version") .description("show version information") .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); } void printFlags(std::ostream & out) override @@ -68,9 +113,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs void mainWrapped(int argc, char * * argv) { - verbosity = lvlError; - settings.verboseBuild = false; - /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { @@ -89,6 +131,9 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } + verbosity = lvlWarn; + settings.verboseBuild = false; + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); @@ -99,8 +144,24 @@ void mainWrapped(int argc, char * * argv) Finally f([]() { stopProgressBar(); }); - if (isatty(STDERR_FILENO)) - startProgressBar(); + startProgressBar(args.printBuildLogs); + + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits<unsigned int>::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } args.command->prepare(); args.command->run(); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 47caa401d..dea5f0557 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -4,8 +4,8 @@ #include "json.hh" #include "common-args.hh" -#include <iomanip> #include <algorithm> +#include <array> using namespace nix; @@ -13,12 +13,14 @@ struct CmdPathInfo : StorePathsCommand, MixJSON { bool showSize = false; bool showClosureSize = false; + bool humanReadable = false; bool showSigs = false; CmdPathInfo() { mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); + mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); mkFlag(0, "sigs", "show signatures", &showSigs); } @@ -40,6 +42,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON "nix path-info -rS /run/current-system | sort -nk2" }, Example{ + "To show a package's closure size and all its dependencies with human readable sizes:", + "nix path-info -rsSh nixpkgs.rust" + }, + Example{ "To check the existence of a path in a binary cache:", "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" }, @@ -58,6 +64,25 @@ struct CmdPathInfo : StorePathsCommand, MixJSON }; } + void printSize(unsigned long long value) + { + if (!humanReadable) { + std::cout << fmt("\t%11d", value); + return; + } + + static const std::array<char, 9> idents{{ + ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' + }}; + size_t power = 0; + double res = value; + while (res > 1024 && power < idents.size()) { + ++power; + res /= 1024; + } + std::cout << fmt("\t%6.1f%c", res, idents.at(power)); + } + void run(ref<Store> store, Paths storePaths) override { size_t pathLen = 0; @@ -78,13 +103,16 @@ struct CmdPathInfo : StorePathsCommand, MixJSON auto info = store->queryPathInfo(storePath); storePath = info->path; // FIXME: screws up padding - std::cout << storePath << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + std::cout << storePath; + + if (showSize || showClosureSize || showSigs) + std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); if (showSize) - std::cout << '\t' << std::setw(11) << info->narSize; + printSize(info->narSize); if (showClosureSize) - std::cout << '\t' << std::setw(11) << store->getClosureSize(storePath).first; + printSize(store->getClosureSize(storePath).first); if (showSigs) { std::cout << '\t'; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 40b905ba3..c0bcfb0c9 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "sync.hh" #include "store-api.hh" +#include "names.hh" #include <atomic> #include <map> @@ -38,6 +39,7 @@ private: std::map<ActivityType, uint64_t> expectedByType; bool visible = true; ActivityId parent; + std::optional<std::string> name; }; struct ActivitiesByType @@ -60,6 +62,7 @@ private: uint64_t corruptedPaths = 0, untrustedPaths = 0; bool active = true; + bool haveUpdate = true; }; Sync<State> state_; @@ -68,14 +71,21 @@ private: std::condition_variable quitCV, updateCV; + bool printBuildLogs; + bool isTTY; + public: - ProgressBar() + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs) + , isTTY(isTTY) { + state_.lock()->active = isTTY; updateThread = std::thread([&]() { auto state(state_.lock()); while (state->active) { - state.wait(updateCV); + if (!state->haveUpdate) + state.wait(updateCV); draw(*state); state.wait_for(quitCV, std::chrono::milliseconds(50)); } @@ -109,8 +119,14 @@ public: void log(State & state, Verbosity lvl, const std::string & s) { - writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); - draw(state); + if (state.active) { + writeToStderr("\r\e[K" + s + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -141,6 +157,7 @@ public: auto nrRounds = getI(fields, 3); if (nrRounds != 1) i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; } if (type == actSubstitute) { @@ -153,6 +170,14 @@ public: name, sub); } + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + if (type == actQueryPathInfo) { auto name = storePathToName(getS(fields, 0)); i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); @@ -163,7 +188,7 @@ public: || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) i->visible = false; - update(); + update(*state); } /* Check whether an activity has an ancestore with the specified @@ -198,7 +223,7 @@ public: state->its.erase(i); } - update(); + update(*state); } void result(ActivityId act, ResultType type, const std::vector<Field> & fields) override @@ -208,38 +233,46 @@ public: if (type == resFileLinked) { state->filesLinked++; state->bytesLinked += getI(fields, 0); - update(); + update(*state); } - else if (type == resBuildLogLine) { + else if (type == resBuildLogLine || type == resPostBuildLogLine) { auto lastLine = trim(getS(fields, 0)); if (!lastLine.empty()) { auto i = state->its.find(act); assert(i != state->its.end()); ActInfo info = *i->second; - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); - update(); + if (printBuildLogs) { + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); + } } } else if (type == resUntrustedPath) { state->untrustedPaths++; - update(); + update(*state); } else if (type == resCorruptedPath) { state->corruptedPaths++; - update(); + update(*state); } else if (type == resSetPhase) { auto i = state->its.find(act); assert(i != state->its.end()); i->second->phase = getS(fields, 0); - update(); + update(*state); } else if (type == resProgress) { @@ -250,7 +283,7 @@ public: actInfo.expected = getI(fields, 1); actInfo.running = getI(fields, 2); actInfo.failed = getI(fields, 3); - update(); + update(*state); } else if (type == resSetExpected) { @@ -262,17 +295,19 @@ public: state->activitiesByType[type].expected -= j; j = getI(fields, 1); state->activitiesByType[type].expected += j; - update(); + update(*state); } } - void update() + void update(State & state) { + state.haveUpdate = true; updateCV.notify_one(); } void draw(State & state) { + state.haveUpdate = false; if (!state.active) return; std::string line; @@ -333,11 +368,18 @@ public: if (running || done || expected || failed) { if (running) - s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - running / unit, done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); else if (expected != done) - s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, - done / unit, expected / unit); + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); else s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); s = fmt(itemFmt, s); @@ -395,9 +437,9 @@ public: } }; -void startProgressBar() +void startProgressBar(bool printBuildLogs) { - logger = new ProgressBar(); + logger = new ProgressBar(printBuildLogs, isatty(STDERR_FILENO)); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/nix/progress-bar.hh index af8eda5a8..4d61175c2 100644 --- a/src/nix/progress-bar.hh +++ b/src/nix/progress-bar.hh @@ -4,7 +4,7 @@ namespace nix { -void startProgressBar(); +void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 1eb716006..f857b2e89 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,8 +1,24 @@ #include <iostream> #include <cstdlib> +#include <cstring> +#include <climits> #include <setjmp.h> +#ifdef READLINE +#include <readline/history.h> +#include <readline/readline.h> +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include <editline.h> +} +#endif + #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" @@ -15,8 +31,6 @@ #include "command.hh" #include "finally.hh" -#include "src/linenoise/linenoise.h" - namespace nix { #define ESC_RED "\033[31m" @@ -31,6 +45,7 @@ struct NixRepl { string curDir; EvalState state; + Bindings * autoArgs; Strings loadedFiles; @@ -117,19 +132,81 @@ NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store) NixRepl::~NixRepl() { - linenoiseHistorySave(historyFile.c_str()); + write_history(historyFile.c_str()); } - static NixRepl * curRepl; // ugly -static void completionCallback(const char * s, linenoiseCompletions *lc) -{ - /* Otherwise, return all symbols that start with the prefix. */ - for (auto & c : curRepl->completePrefix(s)) - linenoiseAddCompletion(lc, c.c_str()); +static char * completionCallback(char * s, int *match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto *res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto &first = *possible.begin(); + for (auto &p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) ++len; + if (len > 0) { + *match = 1; + auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; } +static int listPossibleCallback(char *s, char ***avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) + throw Error("too many completions"); + + int ac = 0; + char **vp = nullptr; + + auto check = [&](auto *p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **)malloc(possible.size() * sizeof(char*))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} + +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} void NixRepl::mainLoop(const std::vector<std::string> & files) { @@ -142,12 +219,18 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) reloadFiles(); if (!loadedFiles.empty()) std::cout << std::endl; + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; createDirs(dirOf(historyFile)); - linenoiseHistorySetMaxLen(1000); - linenoiseHistoryLoad(historyFile.c_str()); - +#ifndef READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); curRepl = this; - linenoiseSetCompletionCallback(completionCallback); +#ifndef READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif std::string input; @@ -175,7 +258,6 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) // We handled the current input fully, so we should clear it // and read brand new input. - linenoiseHistoryAdd(input.c_str()); input.clear(); std::cout << std::endl; } @@ -184,19 +266,42 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) bool NixRepl::getLine(string & input, const std::string &prompt) { - char * s = linenoise(prompt.c_str()); + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); + char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); - if (!s) { - switch (auto type = linenoiseKeyType()) { - case 1: // ctrl-C - input = ""; - return true; - case 2: // ctrl-D - return false; - default: - throw Error(format("Unexpected linenoise keytype: %1%") % type); - } + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; } + + if (!s) + return false; input += s; input += '\n'; return true; @@ -385,7 +490,7 @@ bool NixRepl::processLine(string line) /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", drvPath}) == 0) { + if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { Derivation drv = readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) @@ -441,8 +546,7 @@ void NixRepl::loadFile(const Path & path) loadedFiles.push_back(path); Value v, v2; state.evalFile(lookupFileArg(state, path), v); - Bindings & bindings(*state.allocBindings(0)); - state.autoCallFunction(bindings, v, v2); + state.autoCallFunction(*autoArgs, v, v2); addAttrsToScope(v2); } @@ -584,30 +688,13 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m for (auto & i : *v.attrs) sorted[i.name] = i.value; - /* If this is a derivation, then don't show the - self-references ("all", "out", etc.). */ - StringSet hidden; - if (isDrv) { - hidden.insert("all"); - Bindings::iterator i = v.attrs->find(state.sOutputs); - if (i == v.attrs->end()) - hidden.insert("out"); - else { - state.forceList(*i->value); - for (unsigned int j = 0; j < i->value->listSize(); ++j) - hidden.insert(state.forceStringNoCtx(*i->value->listElems()[j])); - } - } - for (auto & i : sorted) { if (isVarName(i.first)) str << i.first; else printStringValue(str, i.first.c_str()); str << " = "; - if (hidden.find(i.first) != hidden.end()) - str << "«...»"; - else if (seen.find(i.second) != seen.end()) + if (seen.find(i.second) != seen.end()) str << "«repeated»"; else try { @@ -694,6 +781,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref<Store> store) override { auto repl = std::make_unique<NixRepl>(searchPath, openStore()); + repl->autoArgs = getAutoArgs(repl->state); repl->mainLoop(files); } }; diff --git a/src/nix/run.cc b/src/nix/run.cc index d04e106e0..90b76d666 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -7,11 +7,14 @@ #include "finally.hh" #include "fs-accessor.hh" #include "progress-bar.hh" +#include "affinity.hh" #if __linux__ #include <sys/mount.h> #endif +#include <queue> + using namespace nix; std::string chrootHelperName = "__run_in_chroot"; @@ -121,10 +124,27 @@ struct CmdRun : InstallablesCommand unsetenv(var.c_str()); } + std::unordered_set<Path> done; + std::queue<Path> todo; + for (auto & path : outPaths) todo.push(path); + auto unixPath = tokenizeString<Strings>(getEnv("PATH"), ":"); - for (auto & path : outPaths) - if (accessor->stat(path + "/bin").type != FSAccessor::tMissing) + + while (!todo.empty()) { + Path path = todo.front(); + todo.pop(); + if (!done.insert(path).second) continue; + + if (true) unixPath.push_front(path + "/bin"); + + auto propPath = path + "/nix-support/propagated-user-env-packages"; + if (accessor->stat(propPath).type == FSAccessor::tRegular) { + for (auto & p : tokenizeString<Paths>(readFile(propPath))) + todo.push(p); + } + } + setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); std::string cmd = *command.begin(); @@ -135,6 +155,8 @@ struct CmdRun : InstallablesCommand restoreSignals(); + restoreAffinity(); + /* If this is a diverted store (i.e. its "logical" location (typically /nix/store) differs from its "physical" location (e.g. /home/eelco/nix/store), then run the command in a @@ -177,7 +199,10 @@ void chrootHelper(int argc, char * * argv) uid_t gid = getgid(); if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); /* Bind-mount realStoreDir on /nix/store. If the latter mount point doesn't already exists, we have to create a chroot diff --git a/src/nix/search.cc b/src/nix/search.cc index 4cb1efa79..e086de226 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -173,10 +173,12 @@ struct CmdSearch : SourceExprCommand, MixJSON jsonElem.attr("description", description); } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); results[attrPath] = fmt( "* %s (%s)\n %s\n", wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), - wrap("\e[0;2m", hilite(parsed.fullName, nameMatch, "\e[0;2m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), hilite(description, descriptionMatch, ANSI_NORMAL)); } } diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index e23ae7923..35c44a70c 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,14 +1,18 @@ #include "command.hh" +#include "common-args.hh" #include "store-api.hh" #include "download.hh" #include "eval.hh" #include "attr-path.hh" +#include "names.hh" +#include "progress-bar.hh" using namespace nix; -struct CmdUpgradeNix : StoreCommand +struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; + std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -18,6 +22,12 @@ struct CmdUpgradeNix : StoreCommand .labels({"profile-dir"}) .description("the Nix profile to upgrade") .dest(&profileDir); + + mkFlag() + .longName("nix-store-paths-url") + .labels({"url"}) + .description("URL of the file that contains the store paths of the latest Nix release") + .dest(&storePathsUrl); } std::string name() override @@ -59,6 +69,14 @@ struct CmdUpgradeNix : StoreCommand storePath = getLatestNix(store); } + auto version = DrvName(storePathToName(storePath)).version; + + if (dryRun) { + stopProgressBar(); + printError("would upgrade to version %s", version); + return; + } + { Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath)); store->ensurePath(storePath); @@ -72,11 +90,15 @@ struct CmdUpgradeNix : StoreCommand throw Error("could not verify that '%s' works", program); } + stopProgressBar(); + { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir)); runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); } + + printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); } /* Return the profile in which Nix is installed. */ @@ -98,11 +120,18 @@ struct CmdUpgradeNix : StoreCommand if (hasPrefix(where, "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir; - Path userEnv; + Path profileDir = dirOf(where); + + // Resolve profile to /nix/var/nix/profiles/<name> link. + while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = readLink(profileDir); + + printInfo("found profile '%s'", profileDir); + + Path userEnv = canonPath(profileDir, true); if (baseNameOf(where) != "bin" || - !hasSuffix(userEnv = canonPath(profileDir = dirOf(where), true), "user-environment")) + !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); if (!store->isValidPath(userEnv)) @@ -115,7 +144,7 @@ struct CmdUpgradeNix : StoreCommand Path getLatestNix(ref<Store> store) { // FIXME: use nixos.org? - auto req = DownloadRequest("https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"); + auto req = DownloadRequest(storePathsUrl); auto res = getDownloader()->download(req); auto state = std::make_unique<EvalState>(Strings(), store); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 6540208a8..7ef571561 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand for (auto sig : sigs) { if (sigsSeen.count(sig)) continue; sigsSeen.insert(sig); - if (info->checkSignature(publicKeys, sig)) + if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) validSigs++; } }; diff --git a/src/nlohmann/json.hpp b/src/nlohmann/json.hpp index 5b0b0ea5b..c9af0bed3 100644 --- a/src/nlohmann/json.hpp +++ b/src/nlohmann/json.hpp @@ -1,11 +1,12 @@ /* __ _____ _____ _____ __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.0.1 +| | |__ | | | | | | version 3.5.0 |_____|_____|_____|_|___| https://github.com/nlohmann/json Licensed under the MIT License <http://opensource.org/licenses/MIT>. -Copyright (c) 2013-2017 Niels Lohmann <http://nlohmann.me>. +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -29,42 +30,104 @@ SOFTWARE. #ifndef NLOHMANN_JSON_HPP #define NLOHMANN_JSON_HPP -#include <algorithm> // all_of, copy, fill, find, for_each, generate_n, none_of, remove, reverse, transform -#include <array> // array +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 5 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include <algorithm> // all_of, find, for_each #include <cassert> // assert #include <ciso646> // and, not, or -#include <clocale> // lconv, localeconv -#include <cmath> // isfinite, labs, ldexp, signbit #include <cstddef> // nullptr_t, ptrdiff_t, size_t -#include <cstdint> // int64_t, uint64_t -#include <cstdlib> // abort, strtod, strtof, strtold, strtoul, strtoll, strtoull -#include <cstring> // memcpy, strlen -#include <forward_list> // forward_list -#include <functional> // function, hash, less +#include <functional> // hash, less #include <initializer_list> // initializer_list -#include <iomanip> // hex -#include <iosfwd> // istream, ostream -#include <iterator> // advance, begin, back_inserter, bidirectional_iterator_tag, distance, end, inserter, iterator, iterator_traits, next, random_access_iterator_tag, reverse_iterator -#include <limits> // numeric_limits -#include <locale> // locale -#include <map> // map -#include <memory> // addressof, allocator, allocator_traits, unique_ptr +#include <iosfwd> // istream, ostream +#include <iterator> // random_access_iterator_tag #include <numeric> // accumulate -#include <sstream> // stringstream -#include <string> // getline, stoi, string, to_string -#include <type_traits> // add_pointer, conditional, decay, enable_if, false_type, integral_constant, is_arithmetic, is_base_of, is_const, is_constructible, is_convertible, is_default_constructible, is_enum, is_floating_point, is_integral, is_nothrow_move_assignable, is_nothrow_move_constructible, is_pointer, is_reference, is_same, is_scalar, is_signed, remove_const, remove_cv, remove_pointer, remove_reference, true_type, underlying_type -#include <utility> // declval, forward, make_pair, move, pair, swap -#include <valarray> // valarray +#include <string> // string, stoi, to_string +#include <utility> // declval, forward, move, pair, swap + +// #include <nlohmann/json_fwd.hpp> +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include <cstdint> // int64_t, uint64_t +#include <map> // map +#include <memory> // allocator +#include <string> // string #include <vector> // vector +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template<typename T = void, typename SFINAE = void> +struct adl_serializer; + +template<template<typename U, typename V, typename... Args> class ObjectType = + std::map, + template<typename U, typename... Args> class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template<typename U> class AllocatorType = std::allocator, + template<typename T, typename SFINAE = void> class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template<typename BasicJsonType> +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif + +// #include <nlohmann/detail/macro_scope.hpp> + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + // exclude unsupported compilers -#if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif -#elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40900 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif #endif #endif @@ -90,14 +153,36 @@ SOFTWARE. #endif // allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && not defined(JSON_NOEXCEPTION) +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) #define JSON_THROW(exception) throw exception #define JSON_TRY try #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) #else #define JSON_THROW(exception) std::abort() #define JSON_TRY if(true) #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER #endif // manual branch prediction @@ -118,25 +203,35 @@ SOFTWARE. #endif /*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 */ -namespace nlohmann -{ -template<typename = void, typename = void> -struct adl_serializer; - -// forward declaration of basic_json (required to split the class) -template<template<typename, typename, typename...> class ObjectType = std::map, - template<typename, typename...> class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template<typename> class AllocatorType = std::allocator, - template<typename, typename = void> class JSONSerializer = adl_serializer> -class basic_json; +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template<typename BasicJsonType> \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template<typename BasicJsonType> \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } // Ugly macros to avoid uglier copy-paste when specializing basic_json. They // may be removed in the future once the class is split. @@ -154,17 +249,590 @@ class basic_json; NumberIntegerType, NumberUnsignedType, NumberFloatType, \ AllocatorType, JSONSerializer> +// #include <nlohmann/detail/meta/cpp_future.hpp> + + +#include <ciso646> // not +#include <cstddef> // size_t +#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template<bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; + +template<typename T> +using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template<std::size_t... Ints> +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template<class Sequence1, class Sequence2> +struct merge_and_renumber; + +template<std::size_t... I1, std::size_t... I2> +struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template<std::size_t N> +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template<typename... Ts> +using index_sequence_for = make_index_sequence<sizeof...(Ts)>; + +// dispatch utility (taken from ranges-v3) +template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template<typename T> +struct static_const +{ + static constexpr T value{}; +}; + +template<typename T> +constexpr T static_const<T>::value; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +#include <ciso646> // not +#include <limits> // numeric_limits +#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type +#include <utility> // declval + +// #include <nlohmann/json_fwd.hpp> + +// #include <nlohmann/detail/iterators/iterator_traits.hpp> + + +#include <iterator> // random_access_iterator_tag + +// #include <nlohmann/detail/meta/void_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename ...Ts> struct make_void +{ + using type = void; +}; +template <typename ...Ts> using void_t = typename make_void<Ts...>::type; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +namespace nlohmann +{ +namespace detail +{ +template <typename It, typename = void> +struct iterator_types {}; + +template <typename It> +struct iterator_types < + It, + void_t<typename It::difference_type, typename It::value_type, typename It::pointer, + typename It::reference, typename It::iterator_category >> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template <typename T, typename = void> +struct iterator_traits +{ +}; + +template <typename T> +struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >> + : iterator_types<T> +{ +}; + +template <typename T> +struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} +} + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/meta/detected.hpp> + + +#include <type_traits> + +// #include <nlohmann/detail/meta/void_t.hpp> + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template <class Default, + class AlwaysVoid, + template <class...> class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template <class Default, template <class...> class Op, class... Args> +struct detector<Default, void_t<Op<Args...>>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op<Args...>; +}; + +template <template <class...> class Op, class... Args> +using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t; + +template <template <class...> class Op, class... Args> +using detected_t = typename detector<nonesuch, void, Op, Args...>::type; + +template <class Default, template <class...> class Op, class... Args> +using detected_or = detector<Default, void, Op, Args...>; + +template <class Default, template <class...> class Op, class... Args> +using detected_or_t = typename detected_or<Default, Op, Args...>::type; + +template <class Expected, template <class...> class Op, class... Args> +using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>; + +template <class To, template <class...> class Op, class... Args> +using is_detected_convertible = + std::is_convertible<detected_t<Op, Args...>, To>; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + + +namespace nlohmann +{ /*! -@brief unnamed namespace with internal helper functions +@brief detail namespace with internal helper functions -This namespace collects some functions that could not be defined inside the -@ref basic_json class. +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. @since version 2.1.0 */ namespace detail { +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval<T>()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template<typename> struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template <typename T> +using mapped_type_t = typename T::mapped_type; + +template <typename T> +using key_type_t = typename T::key_type; + +template <typename T> +using value_type_t = typename T::value_type; + +template <typename T> +using difference_type_t = typename T::difference_type; + +template <typename T> +using pointer_t = typename T::pointer; + +template <typename T> +using reference_t = typename T::reference; + +template <typename T> +using iterator_category_t = typename T::iterator_category; + +template <typename T> +using iterator_t = typename T::iterator; + +template <typename T, typename... Args> +using to_json_function = decltype(T::to_json(std::declval<Args>()...)); + +template <typename T, typename... Args> +using from_json_function = decltype(T::from_json(std::declval<Args>()...)); + +template <typename T, typename U> +using get_template_function = decltype(std::declval<T>().template get<U>()); + +// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists +template <typename BasicJsonType, typename T, typename = void> +struct has_from_json : std::false_type {}; + +template <typename BasicJsonType, typename T> +struct has_from_json<BasicJsonType, T, + enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<void, from_json_function, serializer, + const BasicJsonType&, T&>::value; +}; + +// This trait checks if JSONSerializer<T>::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template <typename BasicJsonType, typename T, typename = void> +struct has_non_default_from_json : std::false_type {}; + +template<typename BasicJsonType, typename T> +struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<T, from_json_function, serializer, + const BasicJsonType&>::value; +}; + +// This trait checks if BasicJsonType::json_serializer<T>::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template <typename BasicJsonType, typename T, typename = void> +struct has_to_json : std::false_type {}; + +template <typename BasicJsonType, typename T> +struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<void, to_json_function, serializer, BasicJsonType&, + T>::value; +}; + + +/////////////////// +// is_ functions // +/////////////////// + +template <typename T, typename = void> +struct is_iterator_traits : std::false_type {}; + +template <typename T> +struct is_iterator_traits<iterator_traits<T>> +{ + private: + using traits = iterator_traits<T>; + + public: + static constexpr auto value = + is_detected<value_type_t, traits>::value && + is_detected<difference_type_t, traits>::value && + is_detected<pointer_t, traits>::value && + is_detected<iterator_category_t, traits>::value && + is_detected<reference_t, traits>::value; +}; + +// source: https://stackoverflow.com/a/37193089/4116453 + +template <typename T, typename = void> +struct is_complete_type : std::false_type {}; + +template <typename T> +struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {}; + +template <typename BasicJsonType, typename CompatibleObjectType, + typename = void> +struct is_compatible_object_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleObjectType> +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and + is_detected<key_type_t, CompatibleObjectType>::value >> +{ + + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + std::is_constructible<typename object_t::key_type, + typename CompatibleObjectType::key_type>::value and + std::is_constructible<typename object_t::mapped_type, + typename CompatibleObjectType::mapped_type>::value; +}; + +template <typename BasicJsonType, typename CompatibleObjectType> +struct is_compatible_object_type + : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {}; + +template <typename BasicJsonType, typename ConstructibleObjectType, + typename = void> +struct is_constructible_object_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleObjectType> +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and + is_detected<key_type_t, ConstructibleObjectType>::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and + std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or + (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or + has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value); +}; + +template <typename BasicJsonType, typename ConstructibleObjectType> +struct is_constructible_object_type + : is_constructible_object_type_impl<BasicJsonType, + ConstructibleObjectType> {}; + +template <typename BasicJsonType, typename CompatibleStringType, + typename = void> +struct is_compatible_string_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleStringType> +struct is_compatible_string_type_impl < + BasicJsonType, CompatibleStringType, + enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type, + value_type_t, CompatibleStringType>::value >> +{ + static constexpr auto value = + std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value; +}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_compatible_string_type + : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {}; + +template <typename BasicJsonType, typename ConstructibleStringType, + typename = void> +struct is_constructible_string_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_constructible_string_type_impl < + BasicJsonType, ConstructibleStringType, + enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type, + value_type_t, ConstructibleStringType>::value >> +{ + static constexpr auto value = + std::is_constructible<ConstructibleStringType, + typename BasicJsonType::string_t>::value; +}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_constructible_string_type + : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {}; + +template <typename BasicJsonType, typename CompatibleArrayType, typename = void> +struct is_compatible_array_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleArrayType> +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and + is_detected<iterator_t, CompatibleArrayType>::value and +// This is needed because json_reverse_iterator has a ::iterator type... +// Therefore it is detected as a CompatibleArrayType. +// The real fix would be to have an Iterable concept. + not is_iterator_traits< + iterator_traits<CompatibleArrayType>>::value >> +{ + static constexpr bool value = + std::is_constructible<BasicJsonType, + typename CompatibleArrayType::value_type>::value; +}; + +template <typename BasicJsonType, typename CompatibleArrayType> +struct is_compatible_array_type + : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {}; + +template <typename BasicJsonType, typename ConstructibleArrayType, typename = void> +struct is_constructible_array_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t<std::is_same<ConstructibleArrayType, + typename BasicJsonType::value_type>::value >> + : std::true_type {}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t<not std::is_same<ConstructibleArrayType, + typename BasicJsonType::value_type>::value and + is_detected<value_type_t, ConstructibleArrayType>::value and + is_detected<iterator_t, ConstructibleArrayType>::value and + is_complete_type< + detected_t<value_type_t, ConstructibleArrayType>>::value >> +{ + static constexpr bool value = + // This is needed because json_reverse_iterator has a ::iterator type, + // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`... + // Therefore it is detected as a ConstructibleArrayType. + // The real fix would be to have an Iterable concept. + not is_iterator_traits < + iterator_traits<ConstructibleArrayType >>::value and + + (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or + has_from_json<BasicJsonType, + typename ConstructibleArrayType::value_type>::value or + has_non_default_from_json < + BasicJsonType, typename ConstructibleArrayType::value_type >::value); +}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type + : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType, + typename = void> +struct is_compatible_integer_type_impl : std::false_type {}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t<std::is_integral<RealIntegerType>::value and + std::is_integral<CompatibleNumberIntegerType>::value and + not std::is_same<bool, CompatibleNumberIntegerType>::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits<RealIntegerType>; + using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; + + static constexpr auto value = + std::is_constructible<RealIntegerType, + CompatibleNumberIntegerType>::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type + : is_compatible_integer_type_impl<RealIntegerType, + CompatibleNumberIntegerType> {}; + +template <typename BasicJsonType, typename CompatibleType, typename = void> +struct is_compatible_type_impl: std::false_type {}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t<is_complete_type<CompatibleType>::value >> +{ + static constexpr bool value = + has_to_json<BasicJsonType, CompatibleType>::value; +}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type + : is_compatible_type_impl<BasicJsonType, CompatibleType> {}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/exceptions.hpp> + + +#include <exception> // exception +#include <stdexcept> // runtime_error +#include <string> // to_string + +// #include <nlohmann/detail/input/position_t.hpp> + + +#include <cstddef> // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} +} + + +namespace nlohmann +{ +namespace detail +{ //////////////// // exceptions // //////////////// @@ -248,6 +916,7 @@ json.exception.parse_error.109 | parse error: array index 'one' is not a number json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). @note For an input with n bytes, 1 is the index of the first character and n+1 is the index of the terminating null byte or the end of file. This also @@ -271,15 +940,23 @@ class parse_error : public exception /*! @brief create a parse error exception @param[in] id_ the id of the exception - @param[in] byte_ the byte index where the error occurred (or 0 if the - position cannot be determined) + @param[in] position the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) @param[in] what_arg the explanatory string @return parse_error object */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) { std::string w = exception::name("parse_error", id_) + "parse error" + - (byte_ != 0 ? (" at " + std::to_string(byte_)) : "") + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + ": " + what_arg; return parse_error(id_, byte_, w.c_str()); } @@ -298,6 +975,12 @@ class parse_error : public exception private: parse_error(int id_, std::size_t byte_, const char* what_arg) : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } }; /*! @@ -377,6 +1060,7 @@ json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | @liveexample{The following code shows how a `type_error` exception can be caught.,type_error} @@ -419,6 +1103,9 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | @liveexample{The following code shows how an `out_of_range` exception can be caught.,out_of_range} @@ -481,9 +1168,21 @@ class other_error : public exception private: other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/value_t.hpp> +#include <array> // array +#include <ciso646> // and +#include <cstddef> // size_t +#include <cstdint> // uint8_t +namespace nlohmann +{ +namespace detail +{ /////////////////////////// // JSON type enumeration // /////////////////////////// @@ -512,7 +1211,7 @@ value with the default value for a given type @since version 1.0.0 */ -enum class value_t : uint8_t +enum class value_t : std::uint8_t { null, ///< null value object, ///< object (unordered set of name/value pairs) @@ -537,7 +1236,7 @@ Returns an ordering that is similar to Python: */ inline bool operator<(const value_t lhs, const value_t rhs) noexcept { - static constexpr std::array<uint8_t, 8> order = {{ + static constexpr std::array<std::uint8_t, 8> order = {{ 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */ } @@ -547,80 +1246,573 @@ inline bool operator<(const value_t lhs, const value_t rhs) noexcept const auto r_index = static_cast<std::size_t>(rhs); return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]; } +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/conversions/from_json.hpp> -///////////// -// helpers // -///////////// -template<typename> struct is_basic_json : std::false_type {}; +#include <algorithm> // transform +#include <array> // array +#include <ciso646> // and, not +#include <forward_list> // forward_list +#include <iterator> // inserter, front_inserter, end +#include <map> // map +#include <string> // string +#include <tuple> // tuple, make_tuple +#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include <unordered_map> // unordered_map +#include <utility> // pair, declval +#include <valarray> // valarray -NLOHMANN_BASIC_JSON_TPL_DECLARATION -struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; +// #include <nlohmann/detail/exceptions.hpp> -// alias templates to reduce boilerplate -template<bool B, typename T = void> -using enable_if_t = typename std::enable_if<B, T>::type; +// #include <nlohmann/detail/macro_scope.hpp> -template<typename T> -using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; +// #include <nlohmann/detail/meta/cpp_future.hpp> -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template<std::size_t... Ints> -struct index_sequence +// #include <nlohmann/detail/meta/type_traits.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann { - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept +namespace detail +{ +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_UNLIKELY(not j.is_null())) { - return sizeof...(Ints); + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()))); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t<std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_UNLIKELY(not j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); + } + b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template < + typename BasicJsonType, typename ConstructibleStringType, + enable_if_t < + is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and + not std::is_same<typename BasicJsonType::string_t, + ConstructibleStringType>::value, + int > = 0 > +void from_json(const BasicJsonType& j, ConstructibleStringType& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType, typename EnumType, + enable_if_t<std::is_enum<EnumType>::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type<EnumType>::type val; + get_arithmetic_value(j, val); + e = static_cast<EnumType>(val); +} + +// forward_list doesn't have an insert method +template<typename BasicJsonType, typename T, typename Allocator, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get<T>(); + }); +} + +// valarray doesn't have an insert method +template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray<T>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + l.resize(j.size()); + std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); +} + +template<typename BasicJsonType> +void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/) +{ + arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); +} + +template <typename BasicJsonType, typename T, std::size_t N> +auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, + priority_tag<2> /*unused*/) +-> decltype(j.template get<T>(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get<T>(); + } +} + +template<typename BasicJsonType, typename ConstructibleArrayType> +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()), + j.template get<typename ConstructibleArrayType::value_type>(), + void()) +{ + using std::end; + + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename ConstructibleArrayType::value_type>(); + }); +} + +template <typename BasicJsonType, typename ConstructibleArrayType> +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, + priority_tag<0> /*unused*/) +{ + using std::end; + + std::transform( + j.begin(), j.end(), std::inserter(arr, end(arr)), + [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename ConstructibleArrayType::value_type>(); + }); +} + +template <typename BasicJsonType, typename ConstructibleArrayType, + enable_if_t < + is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and + not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and + not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and + not is_basic_json<ConstructibleArrayType>::value, + int > = 0 > + +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) +-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), +j.template get<typename ConstructibleArrayType::value_type>(), +void()) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + + std::string(j.type_name()))); + } + + from_json_array_impl(j, arr, priority_tag<3> {}); +} + +template<typename BasicJsonType, typename ConstructibleObjectType, + enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) +{ + if (JSON_UNLIKELY(not j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); + } + + auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); + using value_type = typename ConstructibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(obj, obj.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>()); + }); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + case value_t::boolean: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType, typename A1, typename A2> +void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) +{ + p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; +} + +template<typename BasicJsonType, typename Tuple, std::size_t... Idx> +void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/) +{ + t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); +} + +template<typename BasicJsonType, typename... Args> +void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +{ + from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); +} + +template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +struct from_json_fn +{ + template<typename BasicJsonType, typename T> + auto operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); } }; +} // namespace detail -template<class Sequence1, class Sequence2> -struct merge_and_renumber; +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace +{ +constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; +} // namespace +} // namespace nlohmann -template<std::size_t... I1, std::size_t... I2> -struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; +// #include <nlohmann/detail/conversions/to_json.hpp> -template<std::size_t N> -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; +#include <ciso646> // or, and, not +#include <iterator> // begin, end +#include <tuple> // tuple, get +#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include <utility> // move, forward, declval, pair +#include <valarray> // valarray +#include <vector> // vector -template<typename... Ts> -using index_sequence_for = make_index_sequence<sizeof...(Ts)>; +// #include <nlohmann/detail/meta/cpp_future.hpp> -/* -Implementation of two C++17 constructs: conjunction, negation. This is needed -to avoid evaluating all the traits in a condition +// #include <nlohmann/detail/meta/type_traits.hpp> -For example: not std::is_same<void, T>::value and has_value_type<T>::value -will not compile when T = void (on MSVC at least). Whereas -conjunction<negation<std::is_same<void, T>>, has_value_type<T>>::value will -stop evaluating if negation<...>::value == false +// #include <nlohmann/detail/value_t.hpp> -Please note that those constructs must be used with caution, since symbols can -become very long quickly (which can slow down compilation and cause MSVC -internal compiler errors). Only use it when you have to (see example ahead). -*/ -template<class...> struct conjunction : std::true_type {}; -template<class B1> struct conjunction<B1> : B1 {}; -template<class B1, class... Bn> -struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {}; +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> -template<class B> struct negation : std::integral_constant<bool, not B::value> {}; -// dispatch utility (taken from ranges-v3) -template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; +#include <cstddef> // size_t +#include <string> // string, to_string +#include <iterator> // input_iterator_tag +#include <tuple> // tuple_size, get, tuple_element + +// #include <nlohmann/detail/value_t.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename IteratorType> class iteration_proxy_value +{ + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_value; + using pointer = value_type * ; + using reference = value_type & ; + using iterator_category = std::input_iterator_tag; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable std::string array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const std::string empty_str = ""; + + public: + explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {} + + /// dereference operator (needed for range-based for) + iteration_proxy_value& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_value& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_value& o) const noexcept + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_value& o) const noexcept + { + return anchor != o.anchor; + } + + /// return key of the iterator + const std::string& key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + array_index_str = std::to_string(array_index); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } +}; + +/// proxy class for the items() function +template<typename IteratorType> class iteration_proxy +{ + private: + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_value<IteratorType> begin() noexcept + { + return iteration_proxy_value<IteratorType>(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_value<IteratorType> end() noexcept + { + return iteration_proxy_value<IteratorType>(container.end()); + } +}; +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0> +auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key()) +{ + return i.key(); +} +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0> +auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value()) +{ + return i.value(); +} +} // namespace detail +} // namespace nlohmann +// The Addition to the STD Namespace is required to add +// Structured Bindings Support to the iteration_proxy_value class +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +namespace std +{ +template <typename IteratorType> +class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>> + : public std::integral_constant<std::size_t, 2> {}; +template <std::size_t N, typename IteratorType> +class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >> +{ + public: + using type = decltype( + get<N>(std::declval < + ::nlohmann::detail::iteration_proxy_value<IteratorType >> ())); +}; +} + +namespace nlohmann +{ +namespace detail +{ ////////////////// // constructors // ////////////////// @@ -657,6 +1849,16 @@ struct external_constructor<value_t::string> j.m_value = std::move(s); j.assert_invariant(); } + + template<typename BasicJsonType, typename CompatibleStringType, + enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_type = value_t::string; + j.m_value.string = j.template create<typename BasicJsonType::string_t>(str); + j.assert_invariant(); + } }; template<> @@ -783,159 +1985,6 @@ struct external_constructor<value_t::object> } }; - -//////////////////////// -// has_/is_ functions // -//////////////////////// - -/*! -@brief Helper to determine whether there's a key_type for T. - -This helper is used to tell associative containers apart from other containers -such as sequence containers. For instance, `std::map` passes the test as it -contains a `mapped_type`, whereas `std::vector` fails the test. - -@sa http://stackoverflow.com/a/7728728/266378 -@since version 1.0.0, overworked in version 2.0.6 -*/ -#define NLOHMANN_JSON_HAS_HELPER(type) \ - template<typename T> struct has_##type { \ - private: \ - template<typename U, typename = typename U::type> \ - static int detect(U &&); \ - static void detect(...); \ - public: \ - static constexpr bool value = \ - std::is_integral<decltype(detect(std::declval<T>()))>::value; \ - } - -NLOHMANN_JSON_HAS_HELPER(mapped_type); -NLOHMANN_JSON_HAS_HELPER(key_type); -NLOHMANN_JSON_HAS_HELPER(value_type); -NLOHMANN_JSON_HAS_HELPER(iterator); - -#undef NLOHMANN_JSON_HAS_HELPER - - -template<bool B, class RealType, class CompatibleObjectType> -struct is_compatible_object_type_impl : std::false_type {}; - -template<class RealType, class CompatibleObjectType> -struct is_compatible_object_type_impl<true, RealType, CompatibleObjectType> -{ - static constexpr auto value = - std::is_constructible<typename RealType::key_type, typename CompatibleObjectType::key_type>::value and - std::is_constructible<typename RealType::mapped_type, typename CompatibleObjectType::mapped_type>::value; -}; - -template<class BasicJsonType, class CompatibleObjectType> -struct is_compatible_object_type -{ - static auto constexpr value = is_compatible_object_type_impl < - conjunction<negation<std::is_same<void, CompatibleObjectType>>, - has_mapped_type<CompatibleObjectType>, - has_key_type<CompatibleObjectType>>::value, - typename BasicJsonType::object_t, CompatibleObjectType >::value; -}; - -template<typename BasicJsonType, typename T> -struct is_basic_json_nested_type -{ - static auto constexpr value = std::is_same<T, typename BasicJsonType::iterator>::value or - std::is_same<T, typename BasicJsonType::const_iterator>::value or - std::is_same<T, typename BasicJsonType::reverse_iterator>::value or - std::is_same<T, typename BasicJsonType::const_reverse_iterator>::value; -}; - -template<class BasicJsonType, class CompatibleArrayType> -struct is_compatible_array_type -{ - static auto constexpr value = - conjunction<negation<std::is_same<void, CompatibleArrayType>>, - negation<is_compatible_object_type< - BasicJsonType, CompatibleArrayType>>, - negation<std::is_constructible<typename BasicJsonType::string_t, - CompatibleArrayType>>, - negation<is_basic_json_nested_type<BasicJsonType, CompatibleArrayType>>, - has_value_type<CompatibleArrayType>, - has_iterator<CompatibleArrayType>>::value; -}; - -template<bool, typename, typename> -struct is_compatible_integer_type_impl : std::false_type {}; - -template<typename RealIntegerType, typename CompatibleNumberIntegerType> -struct is_compatible_integer_type_impl<true, RealIntegerType, CompatibleNumberIntegerType> -{ - // is there an assert somewhere on overflows? - using RealLimits = std::numeric_limits<RealIntegerType>; - using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; - - static constexpr auto value = - std::is_constructible<RealIntegerType, CompatibleNumberIntegerType>::value and - CompatibleLimits::is_integer and - RealLimits::is_signed == CompatibleLimits::is_signed; -}; - -template<typename RealIntegerType, typename CompatibleNumberIntegerType> -struct is_compatible_integer_type -{ - static constexpr auto value = - is_compatible_integer_type_impl < - std::is_integral<CompatibleNumberIntegerType>::value and - not std::is_same<bool, CompatibleNumberIntegerType>::value, - RealIntegerType, CompatibleNumberIntegerType >::value; -}; - - -// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists -template<typename BasicJsonType, typename T> -struct has_from_json -{ - private: - // also check the return type of from_json - template<typename U, typename = enable_if_t<std::is_same<void, decltype(uncvref_t<U>::from_json( - std::declval<BasicJsonType>(), std::declval<T&>()))>::value>> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype( - detect(std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - -// This trait checks if JSONSerializer<T>::from_json(json const&) exists -// this overload is used for non-default-constructible user-defined-types -template<typename BasicJsonType, typename T> -struct has_non_default_from_json -{ - private: - template<typename U, typename = - enable_if_t<std::is_same<T, decltype(uncvref_t<U>::from_json(std::declval<BasicJsonType>()))>::value>> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype(detect( - std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - -// This trait checks if BasicJsonType::json_serializer<T>::to_json exists -template<typename BasicJsonType, typename T> -struct has_to_json -{ - private: - template<typename U, typename = decltype(uncvref_t<U>::to_json( - std::declval<BasicJsonType&>(), std::declval<T>()))> - static int detect(U&&); - static void detect(...); - - public: - static constexpr bool value = std::is_integral<decltype(detect( - std::declval<typename BasicJsonType::template json_serializer<T, void>>()))>::value; -}; - - ///////////// // to_json // ///////////// @@ -995,10 +2044,14 @@ void to_json(BasicJsonType& j, const std::vector<bool>& e) external_constructor<value_t::array>::construct(j, e); } -template<typename BasicJsonType, typename CompatibleArrayType, - enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value or - std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, - int> = 0> +template <typename BasicJsonType, typename CompatibleArrayType, + enable_if_t<is_compatible_array_type<BasicJsonType, + CompatibleArrayType>::value and + not is_compatible_object_type< + BasicJsonType, CompatibleArrayType>::value and + not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and + not is_basic_json<CompatibleArrayType>::value, + int> = 0> void to_json(BasicJsonType& j, const CompatibleArrayType& arr) { external_constructor<value_t::array>::construct(j, arr); @@ -1006,7 +2059,7 @@ void to_json(BasicJsonType& j, const CompatibleArrayType& arr) template<typename BasicJsonType, typename T, enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> -void to_json(BasicJsonType& j, std::valarray<T> arr) +void to_json(BasicJsonType& j, const std::valarray<T>& arr) { external_constructor<value_t::array>::construct(j, std::move(arr)); } @@ -1018,7 +2071,7 @@ void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) } template<typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> + enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0> void to_json(BasicJsonType& j, const CompatibleObjectType& obj) { external_constructor<value_t::object>::construct(j, obj); @@ -1030,9 +2083,12 @@ void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) external_constructor<value_t::object>::construct(j, std::move(obj)); } -template<typename BasicJsonType, typename T, std::size_t N, - enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, T (&)[N]>::value, int> = 0> -void to_json(BasicJsonType& j, T (&arr)[N]) +template < + typename BasicJsonType, typename T, std::size_t N, + enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, + const T(&)[N]>::value, + int> = 0 > +void to_json(BasicJsonType& j, const T(&arr)[N]) { external_constructor<value_t::array>::construct(j, arr); } @@ -1040,351 +2096,71 @@ void to_json(BasicJsonType& j, T (&arr)[N]) template<typename BasicJsonType, typename... Args> void to_json(BasicJsonType& j, const std::pair<Args...>& p) { - j = {p.first, p.second}; -} - -template<typename BasicJsonType, typename Tuple, std::size_t... Idx> -void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...>) -{ - j = {std::get<Idx>(t)...}; -} - -template<typename BasicJsonType, typename... Args> -void to_json(BasicJsonType& j, const std::tuple<Args...>& t) -{ - to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); -} - -/////////////// -// from_json // -/////////////// - -// overloads for basic_json template parameters -template<typename BasicJsonType, typename ArithmeticType, - enable_if_t<std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, - int> = 0> -void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) -{ - switch (static_cast<value_t>(j)) - { - case value_t::number_unsigned: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); - break; - } - case value_t::number_integer: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); - break; - } - case value_t::number_float: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); - break; - } - - default: - JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); - } -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) -{ - if (JSON_UNLIKELY(not j.is_boolean())) - { - JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); - } - b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) -{ - if (JSON_UNLIKELY(not j.is_string())) - { - JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); - } - s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) -{ - get_arithmetic_value(j, val); -} - -template<typename BasicJsonType, typename EnumType, - enable_if_t<std::is_enum<EnumType>::value, int> = 0> -void from_json(const BasicJsonType& j, EnumType& e) -{ - typename std::underlying_type<EnumType>::type val; - get_arithmetic_value(j, val); - e = static_cast<EnumType>(val); -} - -template<typename BasicJsonType> -void from_json(const BasicJsonType& j, typename BasicJsonType::array_t& arr) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); -} - -// forward_list doesn't have an insert method -template<typename BasicJsonType, typename T, typename Allocator, - enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> -void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - std::transform(j.rbegin(), j.rend(), - std::front_inserter(l), [](const BasicJsonType & i) - { - return i.template get<T>(); - }); -} - -// valarray doesn't have an insert method -template<typename BasicJsonType, typename T, - enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> -void from_json(const BasicJsonType& j, std::valarray<T>& l) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - l.resize(j.size()); - std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); -} - -template<typename BasicJsonType, typename CompatibleArrayType> -void from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<0> /*unused*/) -{ - using std::end; - - std::transform(j.begin(), j.end(), - std::inserter(arr, end(arr)), [](const BasicJsonType & i) - { - // get<BasicJsonType>() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get<typename CompatibleArrayType::value_type>(); - }); -} - -template<typename BasicJsonType, typename CompatibleArrayType> -auto from_json_array_impl(const BasicJsonType& j, CompatibleArrayType& arr, priority_tag<1> /*unused*/) --> decltype( - arr.reserve(std::declval<typename CompatibleArrayType::size_type>()), - void()) -{ - using std::end; - - arr.reserve(j.size()); - std::transform(j.begin(), j.end(), - std::inserter(arr, end(arr)), [](const BasicJsonType & i) - { - // get<BasicJsonType>() returns *this, this won't call a from_json - // method when value_type is BasicJsonType - return i.template get<typename CompatibleArrayType::value_type>(); - }); -} - -template<typename BasicJsonType, typename T, std::size_t N> -void from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, priority_tag<2> /*unused*/) -{ - for (std::size_t i = 0; i < N; ++i) - { - arr[i] = j.at(i).template get<T>(); - } -} - -template<typename BasicJsonType, typename CompatibleArrayType, - enable_if_t<is_compatible_array_type<BasicJsonType, CompatibleArrayType>::value and - std::is_convertible<BasicJsonType, typename CompatibleArrayType::value_type>::value and - not std::is_same<typename BasicJsonType::array_t, CompatibleArrayType>::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleArrayType& arr) -{ - if (JSON_UNLIKELY(not j.is_array())) - { - JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); - } - - from_json_array_impl(j, arr, priority_tag<2> {}); -} - -template<typename BasicJsonType, typename CompatibleObjectType, - enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value, int> = 0> -void from_json(const BasicJsonType& j, CompatibleObjectType& obj) -{ - if (JSON_UNLIKELY(not j.is_object())) - { - JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); - } - - auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); - using value_type = typename CompatibleObjectType::value_type; - std::transform( - inner_object->begin(), inner_object->end(), - std::inserter(obj, obj.begin()), - [](typename BasicJsonType::object_t::value_type const & p) - { - return value_type(p.first, p.second.template get<typename CompatibleObjectType::mapped_type>()); - }); -} - -// overload for arithmetic types, not chosen for basic_json template arguments -// (BooleanType, etc..); note: Is it really necessary to provide explicit -// overloads for boolean_t etc. in case of a custom BooleanType which is not -// an arithmetic type? -template<typename BasicJsonType, typename ArithmeticType, - enable_if_t < - std::is_arithmetic<ArithmeticType>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and - not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, - int> = 0> -void from_json(const BasicJsonType& j, ArithmeticType& val) -{ - switch (static_cast<value_t>(j)) - { - case value_t::number_unsigned: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); - break; - } - case value_t::number_integer: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); - break; - } - case value_t::number_float: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); - break; - } - case value_t::boolean: - { - val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); - break; - } - - default: - JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); - } + j = { p.first, p.second }; } -template<typename BasicJsonType, typename A1, typename A2> -void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) +// for https://github.com/nlohmann/json/pull/1134 +template < typename BasicJsonType, typename T, + enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0> +void to_json(BasicJsonType& j, const T& b) { - p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; + j = { {b.key(), b.value()} }; } template<typename BasicJsonType, typename Tuple, std::size_t... Idx> -void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...>) +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/) { - t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); + j = { std::get<Idx>(t)... }; } template<typename BasicJsonType, typename... Args> -void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +void to_json(BasicJsonType& j, const std::tuple<Args...>& t) { - from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); + to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); } struct to_json_fn { - private: template<typename BasicJsonType, typename T> - auto call(BasicJsonType& j, T&& val, priority_tag<1> /*unused*/) const noexcept(noexcept(to_json(j, std::forward<T>(val)))) + auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val)))) -> decltype(to_json(j, std::forward<T>(val)), void()) { return to_json(j, std::forward<T>(val)); } +}; +} // namespace detail - template<typename BasicJsonType, typename T> - void call(BasicJsonType& /*unused*/, T&& /*unused*/, priority_tag<0> /*unused*/) const noexcept - { - static_assert(sizeof(BasicJsonType) == 0, - "could not find to_json() method in T's namespace"); +/// namespace to hold default `to_json` function +namespace +{ +constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; +} // namespace +} // namespace nlohmann -#ifdef _MSC_VER - // MSVC does not show a stacktrace for the above assert - using decayed = uncvref_t<T>; - static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, - "forcing MSVC stacktrace to show which T we're talking about."); -#endif - } +// #include <nlohmann/detail/input/input_adapters.hpp> - public: - template<typename BasicJsonType, typename T> - void operator()(BasicJsonType& j, T&& val) const - noexcept(noexcept(std::declval<to_json_fn>().call(j, std::forward<T>(val), priority_tag<1> {}))) - { - return call(j, std::forward<T>(val), priority_tag<1> {}); - } -}; -struct from_json_fn -{ - private: - template<typename BasicJsonType, typename T> - auto call(const BasicJsonType& j, T& val, priority_tag<1> /*unused*/) const - noexcept(noexcept(from_json(j, val))) - -> decltype(from_json(j, val), void()) - { - return from_json(j, val); - } +#include <cassert> // assert +#include <cstddef> // size_t +#include <cstring> // strlen +#include <istream> // istream +#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include <memory> // shared_ptr, make_shared, addressof +#include <numeric> // accumulate +#include <string> // string, char_traits +#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include <utility> // pair, declval +#include <cstdio> //FILE * - template<typename BasicJsonType, typename T> - void call(const BasicJsonType& /*unused*/, T& /*unused*/, priority_tag<0> /*unused*/) const noexcept - { - static_assert(sizeof(BasicJsonType) == 0, - "could not find from_json() method in T's namespace"); -#ifdef _MSC_VER - // MSVC does not show a stacktrace for the above assert - using decayed = uncvref_t<T>; - static_assert(sizeof(typename decayed::force_msvc_stacktrace) == 0, - "forcing MSVC stacktrace to show which T we're talking about."); -#endif - } +// #include <nlohmann/detail/macro_scope.hpp> - public: - template<typename BasicJsonType, typename T> - void operator()(const BasicJsonType& j, T& val) const - noexcept(noexcept(std::declval<from_json_fn>().call(j, val, priority_tag<1> {}))) - { - return call(j, val, priority_tag<1> {}); - } -}; -// taken from ranges-v3 -template<typename T> -struct static_const +namespace nlohmann { - static constexpr T value{}; -}; - -template<typename T> -constexpr T static_const<T>::value; +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; //////////////////// // input adapters // @@ -1394,19 +2170,17 @@ constexpr T static_const<T>::value; @brief abstract input adapter interface Produces a stream of std::char_traits<char>::int_type characters from a -std::istream, a buffer, or some other input type. Accepts the return of exactly -one non-EOF character for future input. The int_type characters returned -consist of all valid char values as positive values (typically unsigned char), -plus an EOF value outside that range, specified by the value of the function -std::char_traits<char>::eof(). This value is typically -1, but could be any -arbitrary value which is not a valid char value. +std::istream, a buffer, or some other input type. Accepts the return of +exactly one non-EOF character for future input. The int_type characters +returned consist of all valid char values as positive values (typically +unsigned char), plus an EOF value outside that range, specified by the value +of the function std::char_traits<char>::eof(). This value is typically -1, but +could be any arbitrary value which is not a valid char value. */ struct input_adapter_protocol { /// get a character [0,255] or std::char_traits<char>::eof(). virtual std::char_traits<char>::int_type get_character() = 0; - /// restore the last non-eof() character to input - virtual void unget_character() = 0; virtual ~input_adapter_protocol() = default; }; @@ -1414,6 +2188,27 @@ struct input_adapter_protocol using input_adapter_t = std::shared_ptr<input_adapter_protocol>; /*! +Input adapter for stdio file access. This adapter read only 1 byte and do not use any + buffer. This adapter is a very low level adapter. +*/ +class file_input_adapter : public input_adapter_protocol +{ + public: + explicit file_input_adapter(std::FILE* f) noexcept + : m_file(f) + {} + + std::char_traits<char>::int_type get_character() noexcept override + { + return std::fgetc(m_file); + } + private: + /// the file pointer to read from + std::FILE* m_file; +}; + + +/*! Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at beginning of input. Does not support changing the underlying std::streambuf in mid-input. Maintains underlying std::istream and std::streambuf to support @@ -1428,56 +2223,32 @@ class input_stream_adapter : public input_adapter_protocol ~input_stream_adapter() override { // clear stream flags; we use underlying streambuf I/O, do not - // maintain ifstream flags - is.clear(); + // maintain ifstream flags, except eof + is.clear(is.rdstate() & std::ios::eofbit); } explicit input_stream_adapter(std::istream& i) : is(i), sb(*i.rdbuf()) - { - // skip byte order mark - std::char_traits<char>::int_type c; - if ((c = get_character()) == 0xEF) - { - if ((c = get_character()) == 0xBB) - { - if ((c = get_character()) == 0xBF) - { - return; // Ignore BOM - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); - } - is.putback('\xBB'); - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); - } - is.putback('\xEF'); - } - else if (c != std::char_traits<char>::eof()) - { - is.unget(); // no byte order mark; process as usual - } - } + {} // delete because of pointer members input_stream_adapter(const input_stream_adapter&) = delete; input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter(input_stream_adapter&&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to // ensure that std::char_traits<char>::eof() and the character 0xFF do not // end up as the same value, eg. 0xFFFFFFFF. std::char_traits<char>::int_type get_character() override { - return sb.sbumpc(); - } - - void unget_character() override - { - sb.sungetc(); // is.unget() avoided for performance + auto res = sb.sbumpc(); + // set eof manually, as we don't use the istream interface. + if (res == EOF) + { + is.clear(is.rdstate() | std::ios::eofbit); + } + return res; } private: @@ -1490,19 +2261,16 @@ class input_stream_adapter : public input_adapter_protocol class input_buffer_adapter : public input_adapter_protocol { public: - input_buffer_adapter(const char* b, const std::size_t l) - : cursor(b), limit(b + l), start(b) - { - // skip byte order mark - if (l >= 3 and b[0] == '\xEF' and b[1] == '\xBB' and b[2] == '\xBF') - { - cursor += 3; - } - } + input_buffer_adapter(const char* b, const std::size_t l) noexcept + : cursor(b), limit(b + l) + {} // delete because of pointer members input_buffer_adapter(const input_buffer_adapter&) = delete; input_buffer_adapter& operator=(input_buffer_adapter&) = delete; + input_buffer_adapter(input_buffer_adapter&&) = delete; + input_buffer_adapter& operator=(input_buffer_adapter&&) = delete; + ~input_buffer_adapter() override = default; std::char_traits<char>::int_type get_character() noexcept override { @@ -1514,28 +2282,182 @@ class input_buffer_adapter : public input_adapter_protocol return std::char_traits<char>::eof(); } - void unget_character() noexcept override + private: + /// pointer to the current character + const char* cursor; + /// pointer past the last character + const char* const limit; +}; + +template<typename WideStringType, size_t T> +struct wide_string_input_helper +{ + // UTF-32 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) { - if (JSON_LIKELY(cursor > start)) + utf8_bytes_index = 0; + + if (current_wchar == str.size()) { - --cursor; + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = static_cast<int>(str[current_wchar++]); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07); + utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } } } +}; + +template<typename WideStringType> +struct wide_string_input_helper<WideStringType, 2> +{ + // UTF-16 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = static_cast<int>(str[current_wchar++]); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6)); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc or wc >= 0xE000) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12)); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else + { + if (current_wchar < str.size()) + { + const auto wc2 = static_cast<int>(str[current_wchar++]); + const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); + utf8_bytes[0] = 0xf0 | (charcode >> 18); + utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (charcode & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + ++current_wchar; + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + } + } + } +}; + +template<typename WideStringType> +class wide_string_input_adapter : public input_adapter_protocol +{ + public: + explicit wide_string_input_adapter(const WideStringType& w) noexcept + : str(w) + {} + + std::char_traits<char>::int_type get_character() noexcept override + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) + { + fill_buffer<sizeof(typename WideStringType::value_type)>(); + + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index == 0); + } + + // use buffer + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; + } private: - /// pointer to the current character - const char* cursor; - /// pointer past the last character - const char* limit; - /// pointer to the first character - const char* start; + template<size_t T> + void fill_buffer() + { + wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + + /// the wstring to process + const WideStringType& str; + + /// index of the current wchar in str + std::size_t current_wchar = 0; + + /// a buffer for UTF-8 bytes + std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; }; class input_adapter { public: // native support - + input_adapter(std::FILE* file) + : ia(std::make_shared<file_input_adapter>(file)) {} /// input adapter for input stream input_adapter(std::istream& i) : ia(std::make_shared<input_stream_adapter>(i)) {} @@ -1544,6 +2466,15 @@ class input_adapter input_adapter(std::istream&& i) : ia(std::make_shared<input_stream_adapter>(i)) {} + input_adapter(const std::wstring& ws) + : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {} + + input_adapter(const std::u16string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {} + + input_adapter(const std::u32string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {} + /// input adapter for buffer template<typename CharT, typename std::enable_if< @@ -1570,23 +2501,26 @@ class input_adapter /// input adapter for iterator range with contiguous storage template<class IteratorType, typename std::enable_if< - std::is_same<typename std::iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value, + std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value, int>::type = 0> input_adapter(IteratorType first, IteratorType last) { +#ifndef NDEBUG // assertion to check that the iterator range is indeed contiguous, // see http://stackoverflow.com/a/35008842/266378 for more discussion - assert(std::accumulate( - first, last, std::pair<bool, int>(true, 0), - [&first](std::pair<bool, int> res, decltype(*first) val) + const auto is_contiguous = std::accumulate( + first, last, std::pair<bool, int>(true, 0), + [&first](std::pair<bool, int> res, decltype(*first) val) { res.first &= (val == *(std::next(std::addressof(*first), res.second++))); return res; - }).first); + }).first; + assert(is_contiguous); +#endif // assertion to check that each element is 1 byte long static_assert( - sizeof(typename std::iterator_traits<IteratorType>::value_type) == 1, + sizeof(typename iterator_traits<IteratorType>::value_type) == 1, "each element in the iterator range must have the size of 1 byte"); const auto len = static_cast<size_t>(std::distance(first, last)); @@ -1610,7 +2544,7 @@ class input_adapter /// input adapter for contiguous container template<class ContiguousContainer, typename std::enable_if<not std::is_pointer<ContiguousContainer>::value and - std::is_base_of<std::random_access_iterator_tag, typename std::iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, + std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, int>::type = 0> input_adapter(const ContiguousContainer& c) : input_adapter(std::begin(c), std::end(c)) {} @@ -1624,10 +2558,34 @@ class input_adapter /// the actual adapter input_adapter_t ia = nullptr; }; +} // namespace detail +} // namespace nlohmann -////////////////////// -// lexer and parser // -////////////////////// +// #include <nlohmann/detail/input/lexer.hpp> + + +#include <clocale> // localeconv +#include <cstddef> // size_t +#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull +#include <cstdio> // snprintf +#include <initializer_list> // initializer_list +#include <string> // char_traits, string +#include <vector> // vector + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/position_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// /*! @brief lexical analysis @@ -1640,6 +2598,7 @@ class lexer using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; public: /// token types for the parser @@ -1701,17 +2660,22 @@ class lexer return "end of input"; case token_type::literal_or_value: return "'[', '{', or a literal"; + // LCOV_EXCL_START default: // catch non-enum values - return "unknown token"; // LCOV_EXCL_LINE + return "unknown token"; + // LCOV_EXCL_STOP } } - explicit lexer(detail::input_adapter_t adapter) + explicit lexer(detail::input_adapter_t&& adapter) : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {} // delete because of pointer members lexer(const lexer&) = delete; + lexer(lexer&&) = delete; lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = delete; + ~lexer() = default; private: ///////////////////// @@ -1819,9 +2783,10 @@ class lexer @brief scan a string literal This function scans a string according to Sect. 7 of RFC 7159. While - scanning, bytes are escaped and copied into buffer yytext. Then the function - returns successfully, yytext is *not* null-terminated (as it may contain \0 - bytes), and yytext.size() is the number of bytes in the string. + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. @return token_type::value_string if string could be successfully scanned, token_type::parse_error otherwise @@ -1831,7 +2796,7 @@ class lexer */ token_type scan_string() { - // reset yytext (ignore opening quote) + // reset token_buffer (ignore opening quote) reset(); // we entered the function by reading an open quote @@ -1999,39 +2964,194 @@ class lexer // invalid control characters case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + case 0x1F: { - error_message = "invalid string: control character must be escaped"; + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; return token_type::parse_error; } @@ -2303,7 +3423,7 @@ class lexer contains cycles, but any cycle can be left when EOF is read. Therefore, the function is guaranteed to terminate. - During scanning, the read bytes are stored in yytext. This string is + During scanning, the read bytes are stored in token_buffer. This string is then converted to a signed integer, an unsigned integer, or a floating-point number. @@ -2315,9 +3435,9 @@ class lexer locale's decimal point is used instead of `.` to work with the locale-dependent converters. */ - token_type scan_number() + token_type scan_number() // lgtm [cpp/use-of-goto] { - // reset yytext to store the number's bytes + // reset token_buffer to store the number's bytes reset(); // the type of the parsed number; initially set to unsigned; will be @@ -2353,11 +3473,13 @@ class lexer goto scan_number_any1; } + // LCOV_EXCL_START default: { // all other characters are rejected outside scan_number() - assert(false); // LCOV_EXCL_LINE + assert(false); } + // LCOV_EXCL_STOP } scan_number_minus: @@ -2601,10 +3723,10 @@ scan_number_done: // try to parse integers first and fall back to floats if (number_type == token_type::value_unsigned) { - const auto x = std::strtoull(yytext.data(), &endptr, 10); + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); if (errno == 0) { @@ -2617,10 +3739,10 @@ scan_number_done: } else if (number_type == token_type::value_integer) { - const auto x = std::strtoll(yytext.data(), &endptr, 10); + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); if (errno == 0) { @@ -2634,10 +3756,10 @@ scan_number_done: // this code is reached if we parse a floating-point number or if an // integer conversion above failed - strtof(value_float, yytext.data(), &endptr); + strtof(value_float, token_buffer.data(), &endptr); // we checked the number format before - assert(endptr == yytext.data() + yytext.size()); + assert(endptr == token_buffer.data() + token_buffer.size()); return token_type::value_float; } @@ -2666,10 +3788,10 @@ scan_number_done: // input management ///////////////////// - /// reset yytext; current character is beginning of token + /// reset token_buffer; current character is beginning of token void reset() noexcept { - yytext.clear(); + token_buffer.clear(); token_string.clear(); token_string.push_back(std::char_traits<char>::to_char_type(current)); } @@ -2686,31 +3808,71 @@ scan_number_done: */ std::char_traits<char>::int_type get() { - ++chars_read; - current = ia->get_character(); + ++position.chars_read_total; + ++position.chars_read_current_line; + + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia->get_character(); + } + if (JSON_LIKELY(current != std::char_traits<char>::eof())) { token_string.push_back(std::char_traits<char>::to_char_type(current)); } + + if (current == '\n') + { + ++position.lines_read; + ++position.chars_read_current_line = 0; + } + return current; } - /// unget current character (return it again on next get) + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. + */ void unget() { - --chars_read; + next_unget = true; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + if (JSON_LIKELY(current != std::char_traits<char>::eof())) { - ia->unget_character(); assert(token_string.size() != 0); token_string.pop_back(); } } - /// add a character to yytext + /// add a character to token_buffer void add(int c) { - yytext.push_back(std::char_traits<char>::to_char_type(c)); + token_buffer.push_back(std::char_traits<char>::to_char_type(c)); } public: @@ -2737,9 +3899,9 @@ scan_number_done: } /// return current string value (implicitly resets the token; useful only once) - std::string move_string() + string_t& get_string() { - return std::move(yytext); + return token_buffer; } ///////////////////// @@ -2747,9 +3909,9 @@ scan_number_done: ///////////////////// /// return position of last read token - constexpr std::size_t get_position() const noexcept + constexpr position_t get_position() const noexcept { - return chars_read; + return position; } /// return the last read token (for errors only). Will never contain EOF @@ -2764,10 +3926,9 @@ scan_number_done: if ('\x00' <= c and c <= '\x1F') { // escape control characters - std::stringstream ss; - ss << "<U+" << std::setw(4) << std::uppercase << std::setfill('0') - << std::hex << static_cast<int>(c) << ">"; - result += ss.str(); + char cs[9]; + (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c)); + result += cs; } else { @@ -2789,8 +3950,33 @@ scan_number_done: // actual scanner ///////////////////// + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + // check if we completely parse the BOM + return get() == 0xBB and get() == 0xBF; + } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + token_type scan() { + // initially, skip the BOM + if (position.chars_read_total == 0 and not skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + // read next character and ignore whitespace do { @@ -2860,14 +4046,17 @@ scan_number_done: /// the current character std::char_traits<char>::int_type current = std::char_traits<char>::eof(); - /// the number of characters read - std::size_t chars_read = 0; + /// whether the next get() call should just return current + bool next_unget = false; + + /// the start position of the current token + position_t position; /// raw input token string (for error messages) std::vector<char> token_string {}; /// buffer for variable-length tokens (numbers, strings) - std::string yytext {}; + string_t token_buffer {}; /// a description of occurred lexer errors const char* error_message = ""; @@ -2880,6 +4069,887 @@ scan_number_done: /// the decimal point const char decimal_point_char = '.'; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/parser.hpp> + + +#include <cassert> // assert +#include <cmath> // isfinite +#include <cstdint> // uint8_t +#include <functional> // function +#include <string> // string +#include <utility> // move + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + + +#include <cstdint> // size_t +#include <utility> // declval + +// #include <nlohmann/detail/meta/detected.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename T> +using null_function_t = decltype(std::declval<T&>().null()); + +template <typename T> +using boolean_function_t = + decltype(std::declval<T&>().boolean(std::declval<bool>())); + +template <typename T, typename Integer> +using number_integer_function_t = + decltype(std::declval<T&>().number_integer(std::declval<Integer>())); + +template <typename T, typename Unsigned> +using number_unsigned_function_t = + decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>())); + +template <typename T, typename Float, typename String> +using number_float_function_t = decltype(std::declval<T&>().number_float( + std::declval<Float>(), std::declval<const String&>())); + +template <typename T, typename String> +using string_function_t = + decltype(std::declval<T&>().string(std::declval<String&>())); + +template <typename T> +using start_object_function_t = + decltype(std::declval<T&>().start_object(std::declval<std::size_t>())); + +template <typename T, typename String> +using key_function_t = + decltype(std::declval<T&>().key(std::declval<String&>())); + +template <typename T> +using end_object_function_t = decltype(std::declval<T&>().end_object()); + +template <typename T> +using start_array_function_t = + decltype(std::declval<T&>().start_array(std::declval<std::size_t>())); + +template <typename T> +using end_array_function_t = decltype(std::declval<T&>().end_array()); + +template <typename T, typename Exception> +using parse_error_function_t = decltype(std::declval<T&>().parse_error( + std::declval<std::size_t>(), std::declval<const std::string&>(), + std::declval<const Exception&>())); + +template <typename SAX, typename BasicJsonType> +struct is_sax +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact<bool, null_function_t, SAX>::value && + is_detected_exact<bool, boolean_function_t, SAX>::value && + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value && + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value && + is_detected_exact<bool, number_float_function_t, SAX, number_float_t, + string_t>::value && + is_detected_exact<bool, string_function_t, SAX, string_t>::value && + is_detected_exact<bool, start_object_function_t, SAX>::value && + is_detected_exact<bool, key_function_t, SAX, string_t>::value && + is_detected_exact<bool, end_object_function_t, SAX>::value && + is_detected_exact<bool, start_array_function_t, SAX>::value && + is_detected_exact<bool, end_array_function_t, SAX>::value && + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value; +}; + +template <typename SAX, typename BasicJsonType> +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact<bool, null_function_t, SAX>::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact<bool, number_float_function_t, SAX, + number_float_t, string_t>::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact<bool, string_function_t, SAX, string_t>::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + + +#include <cstddef> +#include <string> +#include <vector> + +// #include <nlohmann/detail/input/parser.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template<typename BasicJsonType> +struct json_sax +{ + /// type for (signed) integers + using number_integer_t = typename BasicJsonType::number_integer_t; + /// type for unsigned integers + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type for floating-point numbers + using number_float_t = typename BasicJsonType::number_float_t; + /// type for strings + using string_t = typename BasicJsonType::string_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief an floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template<typename BasicJsonType> +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + /*! + @param[in, out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + ref_stack.pop_back(); + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template<typename Value> + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward<Value>(v)); + return &root; + } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v)); + return &(ref_stack.back()->m_value.array->back()); + } + else + { + assert(object_element); + *object_element = BasicJsonType(std::forward<Value>(v)); + return object_element; + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template<typename BasicJsonType> +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep and ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (not ref_stack.empty() and ref_stack.back()) + { + // remove discarded value + if (ref_stack.back()->is_object()) + { + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + // discard array + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (not keep and not ref_stack.empty()) + { + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + } + + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template<typename Value> + std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false) + { + assert(not keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (not keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward<Value>(v)); + + // check callback + const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (not keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + else + { + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (not store_element) + { + return {false, nullptr}; + } + + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// stack to manage which values to keep + std::vector<bool> keep_stack; + /// stack to manage which object keys to keep + std::vector<bool> key_keep_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template<typename BasicJsonType> +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include <nlohmann/detail/input/lexer.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// /*! @brief syntax analysis @@ -2892,6 +4962,7 @@ class parser using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; using lexer_t = lexer<BasicJsonType>; using token_type = typename lexer_t::token_type; @@ -2916,11 +4987,14 @@ class parser std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>; /// a parser reading from an input adapter - explicit parser(detail::input_adapter_t adapter, + explicit parser(detail::input_adapter_t&& adapter, const parser_callback_t cb = nullptr, const bool allow_exceptions_ = true) - : callback(cb), m_lexer(adapter), allow_exceptions(allow_exceptions_) - {} + : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } /*! @brief public parser interface @@ -2934,31 +5008,56 @@ class parser */ void parse(const bool strict, BasicJsonType& result) { - // read first token - get_token(); + if (callback) + { + json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); - parse_internal(true, result); - result.assert_invariant(); + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } - // in strict mode, input must be completely read - if (strict) - { - get_token(); - expect(token_type::end_of_input); - } + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } - // in case of an error, return discarded value - if (errored) - { - result = value_t::discarded; - return; + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } } - - // set top-level value to null if it was discarded by the callback - // function - if (result.is_discarded()) + else { - result = nullptr; + json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } } } @@ -2970,413 +5069,317 @@ class parser */ bool accept(const bool strict = true) { - // read first token - get_token(); + json_sax_acceptor<BasicJsonType> sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } - if (not accept_internal()) + template <typename SAX> + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result and strict and (get_token() != token_type::end_of_input)) { - return false; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); } - // strict => last token must be EOF - return not strict or (get_token() == token_type::end_of_input); + return result; } private: - /*! - @brief the actual parser - @throw parse_error.101 in case of an unexpected token - @throw parse_error.102 if to_unicode fails or surrogate error - @throw parse_error.103 if to_unicode fails - */ - void parse_internal(bool keep, BasicJsonType& result) + template <typename SAX> + bool sax_parse_internal(SAX* sax) { - // never parse after a parse error was detected - assert(not errored); + // stack to remember the hierarchy of structured values we are parsing + // true = array; false = object + std::vector<bool> states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; - // start with a discarded value - if (not result.is_discarded()) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - - switch (last_token) + while (true) { - case token_type::begin_object: + if (not skip_to_state_evaluation) { - if (keep) + // invariant: get_token() was called before each iteration + switch (last_token) { - if (callback) + case token_type::begin_object: { - keep = callback(depth++, parse_event_t::object_start, result); - } + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } - if (not callback or keep) - { - // explicitly set result to object to cope with {} - result.m_type = value_t::object; - result.m_value = value_t::object; - } - } + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + break; + } - // read next token - get_token(); + // parse key + if (JSON_UNLIKELY(last_token != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); + } + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } - // closing } -> we are done - if (last_token == token_type::end_object) - { - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); + } - // parse values - std::string key; - BasicJsonType value; - while (true) - { - // store key - if (not expect(token_type::value_string)) - { - return; + // remember we are now inside an object + states.push_back(false); + + // parse values + get_token(); + continue; } - key = m_lexer.move_string(); - bool keep_tag = false; - if (keep) + case token_type::begin_array: { - if (callback) + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) { - BasicJsonType k(key); - keep_tag = callback(depth, parse_event_t::key, k); + return false; } - else + + // closing ] -> we are done + if (get_token() == token_type::end_array) { - keep_tag = true; + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } + break; } - } - - // parse separator (:) - get_token(); - if (not expect(token_type::name_separator)) - { - return; - } - // parse and add value - get_token(); - value.m_value.destroy(value.m_type); - value.m_type = value_t::discarded; - parse_internal(keep, value); + // remember we are now inside an array + states.push_back(true); - if (JSON_UNLIKELY(errored)) - { - return; + // parse values (no need to call get_token) + continue; } - if (keep and keep_tag and not value.is_discarded()) + case token_type::value_float: { - result.m_value.object->emplace(std::move(key), std::move(value)); - } + const auto res = m_lexer.get_number_float(); - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) - { - get_token(); - continue; + if (JSON_UNLIKELY(not std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'")); + } + else + { + if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + break; + } } - // closing } - if (not expect(token_type::end_object)) + case token_type::literal_false: { - return; + if (JSON_UNLIKELY(not sax->boolean(false))) + { + return false; + } + break; } - break; - } - if (keep and callback and not callback(--depth, parse_event_t::object_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; - } - break; - } - - case token_type::begin_array: - { - if (keep) - { - if (callback) + case token_type::literal_null: { - keep = callback(depth++, parse_event_t::array_start, result); + if (JSON_UNLIKELY(not sax->null())) + { + return false; + } + break; } - if (not callback or keep) + case token_type::literal_true: { - // explicitly set result to array to cope with [] - result.m_type = value_t::array; - result.m_value = value_t::array; + if (JSON_UNLIKELY(not sax->boolean(true))) + { + return false; + } + break; } - } - - // read next token - get_token(); - // closing ] -> we are done - if (last_token == token_type::end_array) - { - if (callback and not callback(--depth, parse_event_t::array_end, result)) + case token_type::value_integer: { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; + if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; } - break; - } - // parse values - BasicJsonType value; - while (true) - { - // parse value - value.m_value.destroy(value.m_type); - value.m_type = value_t::discarded; - parse_internal(keep, value); - - if (JSON_UNLIKELY(errored)) + case token_type::value_string: { - return; + if (JSON_UNLIKELY(not sax->string(m_lexer.get_string()))) + { + return false; + } + break; } - if (keep and not value.is_discarded()) + case token_type::value_unsigned: { - result.m_value.array->push_back(std::move(value)); + if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) + case token_type::parse_error: { - get_token(); - continue; + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::uninitialized, "value"))); } - // closing ] - if (not expect(token_type::end_array)) + default: // the last token was unexpected { - return; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::literal_or_value, "value"))); } - break; - } - - if (keep and callback and not callback(--depth, parse_event_t::array_end, result)) - { - result.m_value.destroy(result.m_type); - result.m_type = value_t::discarded; } - break; - } - - case token_type::literal_null: - { - result.m_type = value_t::null; - break; - } - - case token_type::value_string: - { - result.m_type = value_t::string; - result.m_value = m_lexer.move_string(); - break; - } - - case token_type::literal_true: - { - result.m_type = value_t::boolean; - result.m_value = true; - break; - } - - case token_type::literal_false: - { - result.m_type = value_t::boolean; - result.m_value = false; - break; } - - case token_type::value_unsigned: + else { - result.m_type = value_t::number_unsigned; - result.m_value = m_lexer.get_number_unsigned(); - break; + skip_to_state_evaluation = false; } - case token_type::value_integer: + // we reached this line after we successfully parsed a value + if (states.empty()) { - result.m_type = value_t::number_integer; - result.m_value = m_lexer.get_number_integer(); - break; + // empty stack: we reached the end of the hierarchy: done + return true; } - - case token_type::value_float: + else { - result.m_type = value_t::number_float; - result.m_value = m_lexer.get_number_float(); - - // throw in case of infinity or NAN - if (JSON_UNLIKELY(not std::isfinite(result.m_value.number_float))) + if (states.back()) // array { - if (allow_exceptions) + // comma -> next value + if (get_token() == token_type::value_separator) { - JSON_THROW(out_of_range::create(406, "number overflow parsing '" + - m_lexer.get_token_string() + "'")); + // parse a new value + get_token(); + continue; } - expect(token_type::uninitialized); - } - break; - } - - case token_type::parse_error: - { - // using "uninitialized" to avoid "expected" message - if (not expect(token_type::uninitialized)) - { - return; - } - break; // LCOV_EXCL_LINE - } - - default: - { - // the last token was unexpected; we expected a value - if (not expect(token_type::literal_or_value)) - { - return; - } - break; // LCOV_EXCL_LINE - } - } - - if (keep and callback and not callback(depth, parse_event_t::value, result)) - { - result.m_type = value_t::discarded; - } - } - - /*! - @brief the actual acceptor - - @invariant 1. The last token is not yet processed. Therefore, the caller - of this function must make sure a token has been read. - 2. When this function returns, the last token is processed. - That is, the last read character was already considered. - - This invariant makes sure that no token needs to be "unput". - */ - bool accept_internal() - { - switch (last_token) - { - case token_type::begin_object: - { - // read next token - get_token(); - // closing } -> we are done - if (last_token == token_type::end_object) - { - return true; - } - - // parse values - while (true) - { - // parse key - if (last_token != token_type::value_string) + // closing ] + if (JSON_LIKELY(last_token == token_type::end_array)) { - return false; - } + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } - // parse separator (:) - get_token(); - if (last_token != token_type::name_separator) - { - return false; + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; } - - // parse value - get_token(); - if (not accept_internal()) + else { - return false; + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_array, "array"))); } - + } + else // object + { // comma -> next value - get_token(); - if (last_token == token_type::value_separator) + if (get_token() == token_type::value_separator) { + // parse key + if (JSON_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); + } + else + { + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } + } + + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); + } + + // parse values get_token(); continue; } // closing } - return (last_token == token_type::end_object); - } - } - - case token_type::begin_array: - { - // read next token - get_token(); - - // closing ] -> we are done - if (last_token == token_type::end_array) - { - return true; - } - - // parse values - while (true) - { - // parse value - if (not accept_internal()) + if (JSON_LIKELY(last_token == token_type::end_object)) { - return false; - } + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } - // comma -> next value - get_token(); - if (last_token == token_type::value_separator) - { - get_token(); + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; continue; } - - // closing ] - return (last_token == token_type::end_array); + else + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_object, "object"))); + } } } - - case token_type::value_float: - { - // reject infinity or NAN - return std::isfinite(m_lexer.get_number_float()); - } - - case token_type::literal_false: - case token_type::literal_null: - case token_type::literal_true: - case token_type::value_integer: - case token_type::value_string: - case token_type::value_unsigned: - return true; - - default: // the last token was unexpected - return false; } } @@ -3386,31 +5389,17 @@ class parser return (last_token = m_lexer.scan()); } - /*! - @throw parse_error.101 if expected token did not occur - */ - bool expect(token_type t) + std::string exception_message(const token_type expected, const std::string& context) { - if (JSON_UNLIKELY(t != last_token)) + std::string error_msg = "syntax error "; + + if (not context.empty()) { - errored = true; - expected = t; - if (allow_exceptions) - { - throw_exception(); - } - else - { - return false; - } + error_msg += "while parsing " + context + " "; } - return true; - } + error_msg += "- "; - [[noreturn]] void throw_exception() const - { - std::string error_msg = "syntax error - "; if (last_token == token_type::parse_error) { error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + @@ -3426,31 +5415,33 @@ class parser error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); } - JSON_THROW(parse_error::create(101, m_lexer.get_position(), error_msg)); + return error_msg; } private: - /// current level of recursion - int depth = 0; /// callback function const parser_callback_t callback = nullptr; /// the type of the last read token token_type last_token = token_type::uninitialized; /// the lexer lexer_t m_lexer; - /// whether a syntax error occurred - bool errored = false; - /// possible reason for the syntax error - token_type expected = token_type::uninitialized; /// whether to throw exceptions in case of errors const bool allow_exceptions = true; }; +} // namespace detail +} // namespace nlohmann -/////////////// -// iterators // -/////////////// +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> -/*! + +#include <cstddef> // ptrdiff_t +#include <limits> // numeric_limits + +namespace nlohmann +{ +namespace detail +{ +/* @brief an iterator for primitive JSON types This class models an iterator for primitive JSON types (boolean, number, @@ -3461,9 +5452,15 @@ end_value (`1`) models past the end. */ class primitive_iterator_t { - public: + private: using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); + + public: constexpr difference_type get_value() const noexcept { return m_it; @@ -3503,10 +5500,10 @@ class primitive_iterator_t return lhs.m_it < rhs.m_it; } - primitive_iterator_t operator+(difference_type i) + primitive_iterator_t operator+(difference_type n) noexcept { auto result = *this; - result += i; + result += n; return result; } @@ -3515,57 +5512,57 @@ class primitive_iterator_t return lhs.m_it - rhs.m_it; } - friend std::ostream& operator<<(std::ostream& os, primitive_iterator_t it) - { - return os << it.m_it; - } - - primitive_iterator_t& operator++() + primitive_iterator_t& operator++() noexcept { ++m_it; return *this; } - primitive_iterator_t const operator++(int) + primitive_iterator_t const operator++(int) noexcept { auto result = *this; - m_it++; + ++m_it; return result; } - primitive_iterator_t& operator--() + primitive_iterator_t& operator--() noexcept { --m_it; return *this; } - primitive_iterator_t const operator--(int) + primitive_iterator_t const operator--(int) noexcept { auto result = *this; - m_it--; + --m_it; return result; } - primitive_iterator_t& operator+=(difference_type n) + primitive_iterator_t& operator+=(difference_type n) noexcept { m_it += n; return *this; } - primitive_iterator_t& operator-=(difference_type n) + primitive_iterator_t& operator-=(difference_type n) noexcept { m_it -= n; return *this; } +}; +} // namespace detail +} // namespace nlohmann - private: - static constexpr difference_type begin_value = 0; - static constexpr difference_type end_value = begin_value + 1; +// #include <nlohmann/detail/iterators/internal_iterator.hpp> + + +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> - /// iterator as signed integer type - difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); -}; +namespace nlohmann +{ +namespace detail +{ /*! @brief an iterator value @@ -3581,26 +5578,50 @@ template<typename BasicJsonType> struct internal_iterator /// generic iterator for all other types primitive_iterator_t primitive_iterator {}; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/iterators/iter_impl.hpp> + + +#include <ciso646> // not +#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include <type_traits> // conditional, is_const, remove_const + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/iterators/internal_iterator.hpp> +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on template<typename IteratorType> class iteration_proxy; +template<typename IteratorType> class iteration_proxy_value; /*! @brief a template for a bidirectional iterator for the @ref basic_json class - This class implements a both iterators (iterator and const_iterator) for the @ref basic_json class. - @note An iterator is called *initialized* when a pointer to a JSON value has been set (e.g., by a constructor or a copy assignment). If the iterator is default-constructed, it is *uninitialized* and most methods are undefined. **The library uses assertions to detect calls on uninitialized iterators.** - @requirement The class satisfies the following concept requirements: - -[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator): +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): The iterator that can be moved can be moved in both directions (i.e. incremented and decremented). - @since version 1.0.0, simplified in version 2.0.9, change to bidirectional iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) */ @@ -3611,6 +5632,7 @@ class iter_impl friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>; friend BasicJsonType; friend iteration_proxy<iter_impl>; + friend iteration_proxy_value<iter_impl>; using object_t = typename BasicJsonType::object_t; using array_t = typename BasicJsonType::array_t; @@ -4149,7 +6171,7 @@ class iter_impl @brief return the key of an object iterator @pre The iterator is initialized; i.e. `m_object != nullptr`. */ - typename object_t::key_type key() const + const typename object_t::key_type& key() const { assert(m_object != nullptr); @@ -4174,94 +6196,26 @@ class iter_impl /// associated JSON instance pointer m_object = nullptr; /// the actual iterator of the associated instance - internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it = {}; + internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it; }; +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> -/// proxy class for the iterator_wrapper functions -template<typename IteratorType> class iteration_proxy -{ - private: - /// helper class for iteration - class iteration_proxy_internal - { - private: - /// the iterator - IteratorType anchor; - /// an index for arrays (used to create key names) - std::size_t array_index = 0; - - public: - explicit iteration_proxy_internal(IteratorType it) noexcept : anchor(it) {} - - /// dereference operator (needed for range-based for) - iteration_proxy_internal& operator*() - { - return *this; - } - - /// increment operator (needed for range-based for) - iteration_proxy_internal& operator++() - { - ++anchor; - ++array_index; - - return *this; - } - - /// inequality operator (needed for range-based for) - bool operator!=(const iteration_proxy_internal& o) const noexcept - { - return anchor != o.anchor; - } - - /// return key of the iterator - std::string key() const - { - assert(anchor.m_object != nullptr); - - switch (anchor.m_object->type()) - { - // use integer array index as key - case value_t::array: - return std::to_string(array_index); - - // use key from the object - case value_t::object: - return anchor.key(); - - // use an empty key for all primitive types - default: - return ""; - } - } - - /// return value of the iterator - typename IteratorType::reference value() const - { - return anchor.value(); - } - }; - - /// the container to iterate - typename IteratorType::reference container; +// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp> - public: - /// construct iteration proxy from a container - explicit iteration_proxy(typename IteratorType::reference cont) - : container(cont) {} - /// return iterator begin (needed for range-based for) - iteration_proxy_internal begin() noexcept - { - return iteration_proxy_internal(container.begin()); - } +#include <cstddef> // ptrdiff_t +#include <iterator> // reverse_iterator +#include <utility> // declval - /// return iterator end (needed for range-based for) - iteration_proxy_internal end() noexcept - { - return iteration_proxy_internal(container.end()); - } -}; +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// /*! @brief a template for a reverse iterator class @@ -4272,10 +6226,10 @@ create @ref const_reverse_iterator). @requirement The class satisfies the following concept requirements: - -[BidirectionalIterator](http://en.cppreference.com/w/cpp/concept/BidirectionalIterator): +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): The iterator that can be moved can be moved in both directions (i.e. incremented and decremented). -- [OutputIterator](http://en.cppreference.com/w/cpp/concept/OutputIterator): +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): It is possible to write to the pointed-to element (only if @a Base is @ref iterator). @@ -4292,11 +6246,11 @@ class json_reverse_iterator : public std::reverse_iterator<Base> using reference = typename Base::reference; /// create reverse iterator from iterator - json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept : base_iterator(it) {} /// create reverse iterator from base class - json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} /// post-increment (it++) json_reverse_iterator const operator++(int) @@ -4366,11 +6320,25 @@ class json_reverse_iterator : public std::reverse_iterator<Base> return it.operator * (); } }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/output_adapters.hpp> -///////////////////// -// output adapters // -///////////////////// +#include <algorithm> // copy +#include <cstddef> // size_t +#include <ios> // streamsize +#include <iterator> // back_inserter +#include <memory> // shared_ptr, make_shared +#include <ostream> // basic_ostream +#include <string> // basic_string +#include <vector> // vector + +namespace nlohmann +{ +namespace detail +{ /// abstract output adapter interface template<typename CharType> struct output_adapter_protocol { @@ -4388,7 +6356,9 @@ template<typename CharType> class output_vector_adapter : public output_adapter_protocol<CharType> { public: - explicit output_vector_adapter(std::vector<CharType>& vec) : v(vec) {} + explicit output_vector_adapter(std::vector<CharType>& vec) noexcept + : v(vec) + {} void write_character(CharType c) override { @@ -4409,7 +6379,9 @@ template<typename CharType> class output_stream_adapter : public output_adapter_protocol<CharType> { public: - explicit output_stream_adapter(std::basic_ostream<CharType>& s) : stream(s) {} + explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept + : stream(s) + {} void write_character(CharType c) override { @@ -4426,11 +6398,13 @@ class output_stream_adapter : public output_adapter_protocol<CharType> }; /// output adapter for basic_string -template<typename CharType> +template<typename CharType, typename StringType = std::basic_string<CharType>> class output_string_adapter : public output_adapter_protocol<CharType> { public: - explicit output_string_adapter(std::basic_string<CharType>& s) : str(s) {} + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} void write_character(CharType c) override { @@ -4443,10 +6417,10 @@ class output_string_adapter : public output_adapter_protocol<CharType> } private: - std::basic_string<CharType>& str; + StringType& str; }; -template<typename CharType> +template<typename CharType, typename StringType = std::basic_string<CharType>> class output_adapter { public: @@ -4456,8 +6430,8 @@ class output_adapter output_adapter(std::basic_ostream<CharType>& s) : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {} - output_adapter(std::basic_string<CharType>& s) - : oa(std::make_shared<output_string_adapter<CharType>>(s)) {} + output_adapter(StringType& s) + : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {} operator output_adapter_t<CharType>() { @@ -4467,19 +6441,57 @@ class output_adapter private: output_adapter_t<CharType> oa = nullptr; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/binary_reader.hpp> -////////////////////////////// -// binary reader and writer // -////////////////////////////// + +#include <algorithm> // generate_n +#include <array> // array +#include <cassert> // assert +#include <cmath> // ldexp +#include <cstddef> // size_t +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstdio> // snprintf +#include <cstring> // memcpy +#include <iterator> // back_inserter +#include <limits> // numeric_limits +#include <string> // char_traits, string +#include <utility> // make_pair, move + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary reader // +/////////////////// /*! -@brief deserialization of CBOR and MessagePack values +@brief deserialization of CBOR, MessagePack, and UBJSON values */ -template<typename BasicJsonType> +template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>> class binary_reader { using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using json_sax_t = SAX; public: /*! @@ -4489,49 +6501,68 @@ class binary_reader */ explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; assert(ia); } /*! - @brief create a JSON value from CBOR input - + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor @param[in] strict whether to expect the input to be consumed completed - @return JSON value created from CBOR input - @throw parse_error.110 if input ended unexpectedly or the end of file was - not reached when @a strict was set to true - @throw parse_error.112 if unsupported byte was read + @return */ - BasicJsonType parse_cbor(const bool strict) + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true) { - const auto res = parse_cbor_internal(); - if (strict) + sax = sax_; + bool result = false; + + switch (format) { - get(); - check_eof(true); - } - return res; - } + case input_format_t::bson: + result = parse_bson_internal(); + break; - /*! - @brief create a JSON value from MessagePack input + case input_format_t::cbor: + result = parse_cbor_internal(); + break; - @param[in] strict whether to expect the input to be consumed completed - @return JSON value created from MessagePack input + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; - @throw parse_error.110 if input ended unexpectedly or the end of file was - not reached when @a strict was set to true - @throw parse_error.112 if unsupported byte was read - */ - BasicJsonType parse_msgpack(const bool strict) - { - const auto res = parse_msgpack_internal(); - if (strict) + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP + } + + // strict mode: next byte must be EOF + if (result and strict) { - get(); - check_eof(true); + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); + } } - return res; + + return result; } /*! @@ -4547,18 +6578,239 @@ class binary_reader } private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast<char>(current); + } + + return true; + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template<typename NumberType> + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + } + + return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof(); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const int element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number; + return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0x02: // string + { + std::int32_t len; + string_t value; + return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value; + return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value; + return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + char cr[3]; + (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type)); + return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr))); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + while (int element_type = get()) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_UNLIKELY(not get_bson_cstr(key))) + { + return false; + } + + if (not is_array) + { + if (not sax->key(key)) + { + return false; + } + } + + if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + /*! @param[in] get_char whether a new character should be retrieved from the input (true, default) or whether the last read character should be considered instead + + @return whether a valid CBOR value was passed to the SAX parser */ - BasicJsonType parse_cbor_internal(const bool get_char = true) + bool parse_cbor_internal(const bool get_char = true) { switch (get_char ? get() : current) { // EOF case std::char_traits<char>::eof(): - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return unexpect_eof(input_format_t::cbor, "value"); // Integer 0x00..0x17 (0..23) case 0x00: @@ -4585,19 +6837,31 @@ class binary_reader case 0x15: case 0x16: case 0x17: - return static_cast<number_unsigned_t>(current); + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); case 0x18: // Unsigned integer (one-byte uint8_t follows) - return get_number<uint8_t>(); + { + uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } case 0x19: // Unsigned integer (two-byte uint16_t follows) - return get_number<uint16_t>(); + { + uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } case 0x1A: // Unsigned integer (four-byte uint32_t follows) - return get_number<uint32_t>(); + { + uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } case 0x1B: // Unsigned integer (eight-byte uint64_t follows) - return get_number<uint64_t>(); + { + uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } // Negative integer -1-0x00..-1-0x17 (-1..-24) case 0x20: @@ -4624,28 +6888,31 @@ class binary_reader case 0x35: case 0x36: case 0x37: - return static_cast<int8_t>(0x20 - 1 - current); + return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current)); case 0x38: // Negative integer (one-byte uint8_t follows) { - // must be uint8_t ! - return static_cast<number_integer_t>(-1) - get_number<uint8_t>(); + uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x39: // Negative integer -1-n (two-byte uint16_t follows) { - return static_cast<number_integer_t>(-1) - get_number<uint16_t>(); + uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) { - return static_cast<number_integer_t>(-1) - get_number<uint32_t>(); + uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); } case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) { - return static_cast<number_integer_t>(-1) - - static_cast<number_integer_t>(get_number<uint64_t>()); + uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) + - static_cast<number_integer_t>(number)); } // UTF-8 string (0x00..0x17 bytes follow) @@ -4679,7 +6946,8 @@ class binary_reader case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) case 0x7F: // UTF-8 string (indefinite length) { - return get_cbor_string(); + string_t s; + return get_cbor_string(s) and sax->string(s); } // array (0x00..0x17 data items follow) @@ -4707,39 +6975,34 @@ class binary_reader case 0x95: case 0x96: case 0x97: - { - return get_cbor_array(current & 0x1F); - } + return get_cbor_array(static_cast<std::size_t>(current & 0x1F)); case 0x98: // array (one-byte uint8_t for n follows) { - return get_cbor_array(get_number<uint8_t>()); + uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x99: // array (two-byte uint16_t for n follow) { - return get_cbor_array(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9A: // array (four-byte uint32_t for n follow) { - return get_cbor_array(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9B: // array (eight-byte uint64_t for n follow) { - return get_cbor_array(get_number<uint64_t>()); + uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); } case 0x9F: // array (indefinite length) - { - BasicJsonType result = value_t::array; - while (get() != 0xFF) - { - result.push_back(parse_cbor_internal(false)); - } - return result; - } + return get_cbor_array(std::size_t(-1)); // map (0x00..0x17 pairs of data items follow) case 0xA0: @@ -4766,62 +7029,59 @@ class binary_reader case 0xB5: case 0xB6: case 0xB7: - { - return get_cbor_object(current & 0x1F); - } + return get_cbor_object(static_cast<std::size_t>(current & 0x1F)); case 0xB8: // map (one-byte uint8_t for n follows) { - return get_cbor_object(get_number<uint8_t>()); + uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xB9: // map (two-byte uint16_t for n follow) { - return get_cbor_object(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBA: // map (four-byte uint32_t for n follow) { - return get_cbor_object(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBB: // map (eight-byte uint64_t for n follow) { - return get_cbor_object(get_number<uint64_t>()); + uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); } case 0xBF: // map (indefinite length) - { - BasicJsonType result = value_t::object; - while (get() != 0xFF) - { - auto key = get_cbor_string(); - result[key] = parse_cbor_internal(); - } - return result; - } + return get_cbor_object(std::size_t(-1)); case 0xF4: // false - { - return false; - } + return sax->boolean(false); case 0xF5: // true - { - return true; - } + return sax->boolean(true); case 0xF6: // null - { - return value_t::null; - } + return sax->null(); case 0xF9: // Half-Precision Float (two-byte IEEE 754) { - const int byte1 = get(); - check_eof(); - const int byte2 = get(); - check_eof(); + const int byte1_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const int byte2_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast<unsigned char>(byte1_raw); + const auto byte2 = static_cast<unsigned char>(byte2_raw); // code from RFC 7049, Appendix D, Figure 3: // As half-precision floating-point numbers were only added @@ -4832,51 +7092,244 @@ class binary_reader // half-precision floating-point numbers in the C language // is shown in Fig. 3. const int half = (byte1 << 8) + byte2; - const int exp = (half >> 10) & 0x1F; - const int mant = half & 0x3FF; - double val; - if (exp == 0) - { - val = std::ldexp(mant, -24); - } - else if (exp != 31) + const double val = [&half] { - val = std::ldexp(mant + 1024, exp - 25); - } - else - { - val = (mant == 0) ? std::numeric_limits<double>::infinity() - : std::numeric_limits<double>::quiet_NaN(); - } - return (half & 0x8000) != 0 ? -val : val; + const int exp = (half >> 10) & 0x1F; + const int mant = half & 0x3FF; + assert(0 <= exp and exp <= 32); + assert(0 <= mant and mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits<double>::infinity() + : std::numeric_limits<double>::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000) != 0 + ? static_cast<number_float_t>(-val) + : static_cast<number_float_t>(val), ""); } case 0xFA: // Single-Precision Float (four-byte IEEE 754) { - return get_number<float>(); + float number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); } case 0xFB: // Double-Precision Float (eight-byte IEEE 754) { - return get_number<double>(); + double number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); } default: // anything else (0xFF is handled inside the other types) { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, "error reading CBOR; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); } } } - BasicJsonType parse_msgpack_internal() + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, current & 0x1F, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + uint8_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + uint16_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + uint32_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + uint64_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len) + { + if (not JSON_UNLIKELY(sax->start_object(len))) + { + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() { switch (get()) { // EOF case std::char_traits<char>::eof(): - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + return unexpect_eof(input_format_t::msgpack, "value"); // positive fixint case 0x00: @@ -5007,7 +7460,7 @@ class binary_reader case 0x7D: case 0x7E: case 0x7F: - return static_cast<number_unsigned_t>(current); + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); // fixmap case 0x80: @@ -5026,9 +7479,7 @@ class binary_reader case 0x8D: case 0x8E: case 0x8F: - { - return get_msgpack_object(current & 0x0F); - } + return get_msgpack_object(static_cast<std::size_t>(current & 0x0F)); // fixarray case 0x90: @@ -5047,9 +7498,7 @@ class binary_reader case 0x9D: case 0x9E: case 0x9F: - { - return get_msgpack_array(current & 0x0F); - } + return get_msgpack_array(static_cast<std::size_t>(current & 0x0F)); // fixstr case 0xA0: @@ -5084,73 +7533,113 @@ class binary_reader case 0xBD: case 0xBE: case 0xBF: - return get_msgpack_string(); + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } case 0xC0: // nil - return value_t::null; + return sax->null(); case 0xC2: // false - return false; + return sax->boolean(false); case 0xC3: // true - return true; + return sax->boolean(true); case 0xCA: // float 32 - return get_number<float>(); + { + float number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } case 0xCB: // float 64 - return get_number<double>(); + { + double number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } case 0xCC: // uint 8 - return get_number<uint8_t>(); + { + uint8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xCD: // uint 16 - return get_number<uint16_t>(); + { + uint16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xCE: // uint 32 - return get_number<uint32_t>(); + { + uint32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xCF: // uint 64 - return get_number<uint64_t>(); + { + uint64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } case 0xD0: // int 8 - return get_number<int8_t>(); + { + int8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD1: // int 16 - return get_number<int16_t>(); + { + int16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD2: // int 32 - return get_number<int32_t>(); + { + int32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD3: // int 64 - return get_number<int64_t>(); + { + int64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } case 0xD9: // str 8 case 0xDA: // str 16 case 0xDB: // str 32 - return get_msgpack_string(); + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } case 0xDC: // array 16 { - return get_msgpack_array(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); } case 0xDD: // array 32 { - return get_msgpack_array(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); } case 0xDE: // map 16 { - return get_msgpack_object(get_number<uint16_t>()); + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); } case 0xDF: // map 32 { - return get_msgpack_object(get_number<uint32_t>()); + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); } - // positive fixint + // negative fixint case 0xE0: case 0xE1: case 0xE2: @@ -5183,338 +7672,749 @@ class binary_reader case 0xFD: case 0xFE: case 0xFF: - return static_cast<int8_t>(current); + return sax->number_integer(static_cast<int8_t>(current)); default: // anything else { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(112, chars_read, - "error reading MessagePack; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); } } } /*! - @brief get next character from the input + @brief reads a MessagePack string - This function provides the interface to the used input adapter. It does - not throw in case the input reached EOF, but returns a -'ve valued - `std::char_traits<char>::eof()` in that case. + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. - @return character read from the input + @param[out] result created string + + @return whether string creation completed */ - int get() + bool get_msgpack_string(string_t& result) { - ++chars_read; - return (current = ia->get_character()); - } + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } - /* - @brief read a number from the input + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, current & 0x1F, result); + } - @tparam NumberType the type of the number + case 0xD9: // str 8 + { + uint8_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } - @return number of type @a NumberType + case 0xDA: // str 16 + { + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } - @note This function needs to respect the system's endianess, because - bytes in CBOR and MessagePack are stored in network order (big - endian) and therefore need reordering on little endian systems. + case 0xDB: // str 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); + } + } + } - @throw parse_error.110 if input has less than `sizeof(NumberType)` bytes + /*! + @param[in] len the length of the array + @return whether array creation completed */ - template<typename NumberType> NumberType get_number() + bool get_msgpack_array(const std::size_t len) { - // step 1: read input into array with system's byte order - std::array<uint8_t, sizeof(NumberType)> vec; - for (std::size_t i = 0; i < sizeof(NumberType); ++i) + if (JSON_UNLIKELY(not sax->start_array(len))) { - get(); - check_eof(); + return false; + } - // reverse byte order prior to conversion if necessary - if (is_little_endian) - { - vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current); - } - else + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_msgpack_internal())) { - vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE + return false; } } - // step 2: convert array into number of type T and return - NumberType result; - std::memcpy(&result, vec.data(), sizeof(NumberType)); - return result; + return sax->end_array(); } /*! - @brief create a string by reading characters from the input + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } - @param[in] len number of bytes to read + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) + { + return false; + } - @note We can not reserve @a len bytes for the result, because @a len - may be too large. Usually, @ref check_eof() detects the end of - the input before we run out of string memory. + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + key.clear(); + } - @return string created by reading @a len bytes + return sax->end_object(); + } - @throw parse_error.110 if input has less than @a len bytes + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser */ - template<typename NumberType> - std::string get_string(const NumberType len) + bool parse_ubjson_internal(const bool get_char = true) { - std::string result; - std::generate_n(std::back_inserter(result), len, [this]() - { - get(); - check_eof(); - return static_cast<char>(current); - }); - return result; + return get_ubjson_value(get_char ? get_ignore_noop() : current); } /*! - @brief reads a CBOR string + @brief reads a UBJSON string - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. - Additionally, CBOR's strings with indefinite lengths are supported. + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. - @return string + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead - @throw parse_error.110 if input ended - @throw parse_error.113 if an unexpected byte is read + @return whether string creation completed */ - std::string get_cbor_string() + bool get_ubjson_string(string_t& result, const bool get_char = true) { - check_eof(); + if (get_char) + { + get(); // TODO: may we ignore N here? + } + + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } switch (current) { - // UTF-8 string (0x00..0x17 bytes follow) - case 0x60: - case 0x61: - case 0x62: - case 0x63: - case 0x64: - case 0x65: - case 0x66: - case 0x67: - case 0x68: - case 0x69: - case 0x6A: - case 0x6B: - case 0x6C: - case 0x6D: - case 0x6E: - case 0x6F: - case 0x70: - case 0x71: - case 0x72: - case 0x73: - case 0x74: - case 0x75: - case 0x76: - case 0x77: + case 'U': { - return get_string(current & 0x1F); + uint8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 'i': { - return get_string(get_number<uint8_t>()); + int8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 'I': { - return get_string(get_number<uint16_t>()); + int16_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 'l': { - return get_string(get_number<uint32_t>()); + int32_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 'L': { - return get_string(get_number<uint64_t>()); + int64_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); } - case 0x7F: // UTF-8 string (indefinite length) + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': { - std::string result; - while (get() != 0xFF) + uint8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'i': + { + int8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) { - check_eof(); - result.push_back(static_cast<char>(current)); + return false; } - return result; + result = static_cast<std::size_t>(number); + return true; + } + + case 'I': + { + int16_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'l': + { + int32_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'L': + { + int64_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; } default: { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(113, chars_read, "expected a CBOR string; last byte: 0x" + ss.str())); + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); } } } - template<typename NumberType> - BasicJsonType get_cbor_array(const NumberType len) + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair<std::size_t, int>& result) { - BasicJsonType result = value_t::array; - std::generate_n(std::back_inserter(*result.m_value.array), len, [this]() + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') { - return parse_cbor_internal(); - }); - return result; - } + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } - template<typename NumberType> - BasicJsonType get_cbor_object(const NumberType len) - { - BasicJsonType result = value_t::object; - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - len, [this]() + get_ignore_noop(); + if (JSON_UNLIKELY(current != '#')) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); + } + + return get_ubjson_size_value(result.first); + } + else if (current == '#') { - get(); - auto key = get_cbor_string(); - auto val = parse_cbor_internal(); - return std::make_pair(std::move(key), std::move(val)); - }); - return result; + return get_ubjson_size_value(result.first); + } + return true; } /*! - @brief reads a MessagePack string + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits<char>::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); - This function first reads starting bytes to determine the expected - string length and then copies this number of bytes into a string. + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); - @return string + case 'U': + { + uint8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); + } + + case 'i': + { + int8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } - @throw parse_error.110 if input ended - @throw parse_error.113 if an unexpected byte is read + case 'I': + { + int16_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'l': + { + int32_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'L': + { + int64_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'd': + { + float number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'D': + { + double number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'C': // char + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); + } + string_t s(1, static_cast<char>(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) and sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @return whether array creation completed */ - std::string get_msgpack_string() + bool get_ubjson_array() { - check_eof(); + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } - switch (current) + if (size_and_type.first != string_t::npos) { - // fixstr - case 0xA0: - case 0xA1: - case 0xA2: - case 0xA3: - case 0xA4: - case 0xA5: - case 0xA6: - case 0xA7: - case 0xA8: - case 0xA9: - case 0xAA: - case 0xAB: - case 0xAC: - case 0xAD: - case 0xAE: - case 0xAF: - case 0xB0: - case 0xB1: - case 0xB2: - case 0xB3: - case 0xB4: - case 0xB5: - case 0xB6: - case 0xB7: - case 0xB8: - case 0xB9: - case 0xBA: - case 0xBB: - case 0xBC: - case 0xBD: - case 0xBE: - case 0xBF: + if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) { - return get_string(current & 0x1F); + return false; } - case 0xD9: // str 8 + if (size_and_type.second != 0) { - return get_string(get_number<uint8_t>()); + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; } - case 0xDA: // str 16 + while (current != ']') { - return get_string(get_number<uint16_t>()); + if (JSON_UNLIKELY(not parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); } + } - case 0xDB: // str 32 + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) { - return get_string(get_number<uint32_t>()); + return false; } - default: + if (size_and_type.second != 0) { - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << current; - JSON_THROW(parse_error::create(113, chars_read, - "expected a MessagePack string; last byte: 0x" + ss.str())); + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); } } + + return sax->end_object(); } - template<typename NumberType> - BasicJsonType get_msgpack_array(const NumberType len) + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits<char>::eof()` in that case. + + @return character read from the input + */ + int get() { - BasicJsonType result = value_t::array; - std::generate_n(std::back_inserter(*result.m_value.array), len, [this]() - { - return parse_msgpack_internal(); - }); - return result; + ++chars_read; + return (current = ia->get_character()); } - template<typename NumberType> - BasicJsonType get_msgpack_object(const NumberType len) + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() { - BasicJsonType result = value_t::object; - std::generate_n(std::inserter(*result.m_value.object, - result.m_value.object->end()), - len, [this]() + do { get(); - auto key = get_msgpack_string(); - auto val = parse_msgpack_internal(); - return std::make_pair(std::move(key), std::move(val)); - }); - return result; + } + while (current == 'N'); + + return current; } - /*! - @brief check if input ended - @throw parse_error.110 if input ended + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianess, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. */ - void check_eof(const bool expect_eof = false) const + template<typename NumberType, bool InputIsLittleEndian = false> + bool get_number(const input_format_t format, NumberType& result) { - if (expect_eof) + // step 1: read input into array with system's byte order + std::array<uint8_t, sizeof(NumberType)> vec; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) { - if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) { - JSON_THROW(parse_error::create(110, chars_read, "expected end of input")); + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian && !InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current); + } + else + { + vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE } } - else + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template<typename NumberType> + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success, &format]() { - if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) { - JSON_THROW(parse_error::create(110, chars_read, "unexpected end of input")); + success = false; } + return static_cast<char>(current); + }); + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, "<end of file>", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + char cr[3]; + (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current)); + return std::string{cr}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further contect information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP } + + return error_msg + " " + context + ": " + detail; } private: @@ -5529,7 +8429,34 @@ class binary_reader /// whether we can assume little endianess const bool is_little_endian = little_endianess(); + + /// the SAX parser + json_sax_t* sax = nullptr; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/binary_writer.hpp> + + +#include <algorithm> // reverse +#include <array> // array +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstring> // memcpy +#include <limits> // numeric_limits + +// #include <nlohmann/detail/input/binary_reader.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// /*! @brief serialization to CBOR and MessagePack values @@ -5537,6 +8464,8 @@ class binary_reader template<typename BasicJsonType, typename CharType> class binary_writer { + using string_t = typename BasicJsonType::string_t; + public: /*! @brief create a binary writer @@ -5549,7 +8478,28 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()))); + } + } + } + + /*! + @param[in] j JSON value to serialize */ void write_cbor(const BasicJsonType& j) { @@ -5557,15 +8507,15 @@ class binary_writer { case value_t::null: { - oa->write_character(static_cast<CharType>(0xF6)); + oa->write_character(to_char_type(0xF6)); break; } case value_t::boolean: { oa->write_character(j.m_value.boolean - ? static_cast<CharType>(0xF5) - : static_cast<CharType>(0xF4)); + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -5582,22 +8532,22 @@ class binary_writer } else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x18)); + oa->write_character(to_char_type(0x18)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x19)); + oa->write_character(to_char_type(0x19)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x1A)); + oa->write_character(to_char_type(0x1A)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else { - oa->write_character(static_cast<CharType>(0x1B)); + oa->write_character(to_char_type(0x1B)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } } @@ -5612,22 +8562,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x38)); + oa->write_character(to_char_type(0x38)); write_number(static_cast<uint8_t>(positive_number)); } else if (positive_number <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x39)); + oa->write_character(to_char_type(0x39)); write_number(static_cast<uint16_t>(positive_number)); } else if (positive_number <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x3A)); + oa->write_character(to_char_type(0x3A)); write_number(static_cast<uint32_t>(positive_number)); } else { - oa->write_character(static_cast<CharType>(0x3B)); + oa->write_character(to_char_type(0x3B)); write_number(static_cast<uint64_t>(positive_number)); } } @@ -5642,30 +8592,30 @@ class binary_writer } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x18)); + oa->write_character(to_char_type(0x18)); write_number(static_cast<uint8_t>(j.m_value.number_unsigned)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x19)); + oa->write_character(to_char_type(0x19)); write_number(static_cast<uint16_t>(j.m_value.number_unsigned)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x1A)); + oa->write_character(to_char_type(0x1A)); write_number(static_cast<uint32_t>(j.m_value.number_unsigned)); } else { - oa->write_character(static_cast<CharType>(0x1B)); + oa->write_character(to_char_type(0x1B)); write_number(static_cast<uint64_t>(j.m_value.number_unsigned)); } break; } - case value_t::number_float: // Double-Precision Float + case value_t::number_float: { - oa->write_character(static_cast<CharType>(0xFB)); + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); write_number(j.m_value.number_float); break; } @@ -5678,25 +8628,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0x60 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x78)); + oa->write_character(to_char_type(0x78)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x79)); + oa->write_character(to_char_type(0x79)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x7A)); + oa->write_character(to_char_type(0x7A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { - oa->write_character(static_cast<CharType>(0x7B)); + oa->write_character(to_char_type(0x7B)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5716,25 +8666,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0x80 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0x98)); + oa->write_character(to_char_type(0x98)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0x99)); + oa->write_character(to_char_type(0x99)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0x9A)); + oa->write_character(to_char_type(0x9A)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { - oa->write_character(static_cast<CharType>(0x9B)); + oa->write_character(to_char_type(0x9B)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5755,25 +8705,25 @@ class binary_writer { write_number(static_cast<uint8_t>(0xA0 + N)); } - else if (N <= 0xFF) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { - oa->write_character(static_cast<CharType>(0xB8)); + oa->write_character(to_char_type(0xB8)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { - oa->write_character(static_cast<CharType>(0xB9)); + oa->write_character(to_char_type(0xB9)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { - oa->write_character(static_cast<CharType>(0xBA)); + oa->write_character(to_char_type(0xBA)); write_number(static_cast<uint32_t>(N)); } // LCOV_EXCL_START - else if (N <= 0xFFFFFFFFFFFFFFFF) + else if (N <= (std::numeric_limits<uint64_t>::max)()) { - oa->write_character(static_cast<CharType>(0xBB)); + oa->write_character(to_char_type(0xBB)); write_number(static_cast<uint64_t>(N)); } // LCOV_EXCL_STOP @@ -5793,7 +8743,7 @@ class binary_writer } /*! - @brief[in] j JSON value to serialize + @param[in] j JSON value to serialize */ void write_msgpack(const BasicJsonType& j) { @@ -5801,15 +8751,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(static_cast<CharType>(0xC0)); + oa->write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { oa->write_character(j.m_value.boolean - ? static_cast<CharType>(0xC3) - : static_cast<CharType>(0xC2)); + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -5828,25 +8778,25 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { // uint 8 - oa->write_character(static_cast<CharType>(0xCC)); + oa->write_character(to_char_type(0xCC)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { // uint 16 - oa->write_character(static_cast<CharType>(0xCD)); + oa->write_character(to_char_type(0xCD)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { // uint 32 - oa->write_character(static_cast<CharType>(0xCE)); + oa->write_character(to_char_type(0xCE)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) { // uint 64 - oa->write_character(static_cast<CharType>(0xCF)); + oa->write_character(to_char_type(0xCF)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } } @@ -5861,28 +8811,28 @@ class binary_writer j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) { // int 8 - oa->write_character(static_cast<CharType>(0xD0)); + oa->write_character(to_char_type(0xD0)); write_number(static_cast<int8_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) { // int 16 - oa->write_character(static_cast<CharType>(0xD1)); + oa->write_character(to_char_type(0xD1)); write_number(static_cast<int16_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) { // int 32 - oa->write_character(static_cast<CharType>(0xD2)); + oa->write_character(to_char_type(0xD2)); write_number(static_cast<int32_t>(j.m_value.number_integer)); } else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)()) { // int 64 - oa->write_character(static_cast<CharType>(0xD3)); + oa->write_character(to_char_type(0xD3)); write_number(static_cast<int64_t>(j.m_value.number_integer)); } } @@ -5899,33 +8849,33 @@ class binary_writer else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) { // uint 8 - oa->write_character(static_cast<CharType>(0xCC)); + oa->write_character(to_char_type(0xCC)); write_number(static_cast<uint8_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) { // uint 16 - oa->write_character(static_cast<CharType>(0xCD)); + oa->write_character(to_char_type(0xCD)); write_number(static_cast<uint16_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) { // uint 32 - oa->write_character(static_cast<CharType>(0xCE)); + oa->write_character(to_char_type(0xCE)); write_number(static_cast<uint32_t>(j.m_value.number_integer)); } else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) { // uint 64 - oa->write_character(static_cast<CharType>(0xCF)); + oa->write_character(to_char_type(0xCF)); write_number(static_cast<uint64_t>(j.m_value.number_integer)); } break; } - case value_t::number_float: // float 64 + case value_t::number_float: { - oa->write_character(static_cast<CharType>(0xCB)); + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); write_number(j.m_value.number_float); break; } @@ -5939,22 +8889,22 @@ class binary_writer // fixstr write_number(static_cast<uint8_t>(0xA0 | N)); } - else if (N <= 255) + else if (N <= (std::numeric_limits<uint8_t>::max)()) { // str 8 - oa->write_character(static_cast<CharType>(0xD9)); + oa->write_character(to_char_type(0xD9)); write_number(static_cast<uint8_t>(N)); } - else if (N <= 65535) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // str 16 - oa->write_character(static_cast<CharType>(0xDA)); + oa->write_character(to_char_type(0xDA)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // str 32 - oa->write_character(static_cast<CharType>(0xDB)); + oa->write_character(to_char_type(0xDB)); write_number(static_cast<uint32_t>(N)); } @@ -5974,16 +8924,16 @@ class binary_writer // fixarray write_number(static_cast<uint8_t>(0x90 | N)); } - else if (N <= 0xFFFF) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // array 16 - oa->write_character(static_cast<CharType>(0xDC)); + oa->write_character(to_char_type(0xDC)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 0xFFFFFFFF) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // array 32 - oa->write_character(static_cast<CharType>(0xDD)); + oa->write_character(to_char_type(0xDD)); write_number(static_cast<uint32_t>(N)); } @@ -6004,16 +8954,16 @@ class binary_writer // fixmap write_number(static_cast<uint8_t>(0x80 | (N & 0xF))); } - else if (N <= 65535) + else if (N <= (std::numeric_limits<uint16_t>::max)()) { // map 16 - oa->write_character(static_cast<CharType>(0xDE)); + oa->write_character(to_char_type(0xDE)); write_number(static_cast<uint16_t>(N)); } - else if (N <= 4294967295) + else if (N <= (std::numeric_limits<uint32_t>::max)()) { // map 32 - oa->write_character(static_cast<CharType>(0xDF)); + oa->write_character(to_char_type(0xDF)); write_number(static_cast<uint32_t>(N)); } @@ -6031,25 +8981,745 @@ class binary_writer } } + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + default: + break; + } + } + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name) + { + const auto it = name.find(static_cast<typename string_t::value_type>(0)); + if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, + "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")")); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast<const CharType*>(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number<double, true>(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast<const CharType*>(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) + { + return sizeof(std::int32_t); + } + else + { + return sizeof(std::int64_t); + } + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number<std::int32_t, true>(static_cast<std::int32_t>(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number<std::int64_t, true>(static_cast<std::int64_t>(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const std::uint64_t value) + { + if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number<std::int32_t, true>(static_cast<std::int32_t>(value)); + } + else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number<std::int64_t, true>(static_cast<std::int64_t>(value)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64")); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); + } + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + default: + assert(false); + return 0ul; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + @return The size of the BSON entry + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j.m_value.number_unsigned); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + default: + assert(false); + return; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul, + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template<typename NumberType, typename std::enable_if< + std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template<typename NumberType, typename std::enable_if< + std::is_unsigned<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= (std::numeric_limits<uint8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + } + + // UBJSON: write number (signed integer) + template<typename NumberType, typename std::enable_if< + std::is_signed<NumberType>::value and + not std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast<int8_t>(n)); + } + else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + // LCOV_EXCL_START + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + /* @brief write a number to output input - @param[in] n number of type @a NumberType @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian @note This function needs to respect the system's endianess, because bytes - in CBOR and MessagePack are stored in network order (big endian) and - therefore need reordering on little endian systems. + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. */ - template<typename NumberType> void write_number(NumberType n) + template<typename NumberType, bool OutputIsLittleEndian = false> + void write_number(const NumberType n) { // step 1: write number to array of length NumberType std::array<CharType, sizeof(NumberType)> vec; std::memcpy(vec.data(), &n, sizeof(NumberType)); // step 2: write array to output (with possible reordering) - if (is_little_endian) + if (is_little_endian and not OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary std::reverse(vec.begin(), vec.end()); @@ -6058,6 +9728,47 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See <https://github.com/nlohmann/json/issues/1286> for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast<char*>(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_pod<CharType>::value, "CharType must be POD"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template<typename C = CharType, + enable_if_t<std::is_unsigned<C>::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed<C>::value and + std::is_signed<char>::value and + std::is_same<char, typename std::remove_cv<InputCharType>::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + private: /// whether we can assume little endianess const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess(); @@ -6065,11 +9776,1150 @@ class binary_writer /// the output output_adapter_t<CharType> oa = nullptr; }; +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/output/serializer.hpp> + + +#include <algorithm> // reverse, remove, fill, find, none_of +#include <array> // array +#include <cassert> // assert +#include <ciso646> // and, or +#include <clocale> // localeconv, lconv +#include <cmath> // labs, isfinite, isnan, signbit +#include <cstddef> // size_t, ptrdiff_t +#include <cstdint> // uint8_t +#include <cstdio> // snprintf +#include <limits> // numeric_limits +#include <string> // string +#include <type_traits> // is_same + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/conversions/to_chars.hpp> + + +#include <cassert> // assert +#include <ciso646> // or, and, not +#include <cmath> // signbit, isfinite +#include <cstdint> // intN_t, uintN_t +#include <cstring> // memcpy, memmove + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template <typename Target, typename Source> +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + uint64_t f = 0; + int e = 0; + + constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + assert(x.e == y.e); + assert(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const uint64_t u_lo = x.f & 0xFFFFFFFF; + const uint64_t u_hi = x.f >> 32; + const uint64_t v_lo = y.f & 0xFFFFFFFF; + const uint64_t v_hi = y.f >> 32; + + const uint64_t p0 = u_lo * v_lo; + const uint64_t p1 = u_lo * v_hi; + const uint64_t p2 = u_hi * v_lo; + const uint64_t p3 = u_hi * v_hi; + + const uint64_t p0_hi = p0 >> 32; + const uint64_t p1_lo = p1 & 0xFFFFFFFF; + const uint64_t p1_hi = p1 >> 32; + const uint64_t p2_lo = p2 & 0xFFFFFFFF; + const uint64_t p2_hi = p2 >> 32; + + uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += uint64_t{1} << (64 - 32 - 1); // round, ties up + + const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + assert(x.f != 0); + + while ((x.f >> 63) == 0) + { + x.f <<= 1; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + assert(delta >= 0); + assert(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template <typename FloatType> +boundaries compute_boundaries(FloatType value) +{ + assert(std::isfinite(value)); + assert(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits<FloatType>::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type; + + const uint64_t bits = reinterpret_bits<bits_type>(value); + const uint64_t E = bits >> (kPrecision - 1); + const uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = (E == 0); + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = (F == 0 and E > 1); + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersSize = 79; + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr cached_power kCachedPowers[] = + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + assert(e >= -1500); + assert(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + assert(index >= 0); + assert(index < kCachedPowersSize); + static_cast<void>(kCachedPowersSize); // Fix warning. + + const cached_power cached = kCachedPowers[index]; + assert(kAlpha <= cached.e + e + 64); + assert(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + else if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + else if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + else if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + else if (n >= 100000) + { + pow10 = 100000; + return 6; + } + else if (n >= 10000) + { + pow10 = 10000; + return 5; + } + else if (n >= 1000) + { + pow10 = 1000; + return 4; + } + else if (n >= 100) + { + pow10 = 100; + return 3; + } + else if (n >= 10) + { + pow10 = 10; + return 2; + } + else + { + pow10 = 1; + return 1; + } +} + +inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta, + uint64_t rest, uint64_t ten_k) +{ + assert(len >= 1); + assert(dist <= delta); + assert(rest <= delta); + assert(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + and delta - rest >= ten_k + and (rest + ten_k < dist or dist - rest > rest + ten_k - dist)) + { + assert(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + assert(M_plus.e >= kAlpha); + assert(M_plus.e <= kGamma); + + uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + assert(p1 > 0); + + uint32_t pow10; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const uint64_t rest = (uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const uint64_t ten_n = uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + assert(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + assert(p2 <= UINT64_MAX / 10); + p2 *= 10; + const uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + assert(m_plus.e == m_minus.e); + assert(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template <typename FloatType> +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3, + "internal error: not enough precision"); + + assert(std::isfinite(value)); + assert(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast<double>(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +inline char* append_exponent(char* buf, int e) +{ + assert(e > -1000); + assert(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast<uint32_t>(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast<char>('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + else + { + *buf++ = static_cast<char>('0' + k / 100); + k %= 100; + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + assert(min_exp < 0); + assert(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n and n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast<size_t>(n - k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (n + 2); + } + + if (0 < n and n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + assert(k > n); + + std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n)); + buf[n] = '.'; + return buf + (k + 1); + } + + if (min_exp < n and n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast<size_t>(-n)); + return buf + (2 + (-n) + k); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1)); + buf[1] = '.'; + buf += 1 + k; + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template <typename FloatType> +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast<void>(last); // maybe unused - fix warning + assert(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + + assert(last - first >= std::numeric_limits<FloatType>::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + assert(len <= std::numeric_limits<FloatType>::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10; + + assert(last - first >= kMaxExp + 2); + assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10); + assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/output/binary_writer.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ /////////////////// // serialization // /////////////////// +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + template<typename BasicJsonType> class serializer { @@ -6077,20 +10927,32 @@ class serializer using number_float_t = typename BasicJsonType::number_float_t; using number_integer_t = typename BasicJsonType::number_integer_t; using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + static constexpr uint8_t UTF8_ACCEPT = 0; + static constexpr uint8_t UTF8_REJECT = 1; + public: /*! @param[in] s output stream to serialize to @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors */ - serializer(output_adapter_t<char> s, const char ichar) - : o(std::move(s)), loc(std::localeconv()), - thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)), - decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)), - indent_char(ichar), indent_string(512, indent_char) {} + serializer(output_adapter_t<char> s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)) + , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} // delete because of pointer members serializer(const serializer&) = delete; serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; /*! @brief internal implementation of the serialization function @@ -6303,323 +11165,249 @@ class serializer private: /*! - @brief returns the number of expected bytes following in UTF-8 string - - @param[in] u the first byte of a UTF-8 string - @return the number of expected bytes following - */ - static constexpr std::size_t bytes_following(const uint8_t u) - { - return ((u <= 127) ? 0 - : ((192 <= u and u <= 223) ? 1 - : ((224 <= u and u <= 239) ? 2 - : ((240 <= u and u <= 247) ? 3 : std::string::npos)))); - } + @brief dump escaped string - /*! - @brief calculates the extra space to escape a JSON string + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. @param[in] s the string to escape @param[in] ensure_ascii whether to escape non-ASCII characters with \uXXXX sequences - @return the number of characters required to escape string @a s @complexity Linear in the length of string @a s. */ - static std::size_t extra_space(const string_t& s, - const bool ensure_ascii) noexcept + void dump_escaped(const string_t& s, const bool ensure_ascii) { - std::size_t res = 0; + uint32_t codepoint; + uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; for (std::size_t i = 0; i < s.size(); ++i) { - switch (s[i]) - { - // control characters that can be escaped with a backslash - case '"': - case '\\': - case '\b': - case '\f': - case '\n': - case '\r': - case '\t': - { - // from c (1 byte) to \x (2 bytes) - res += 1; - break; - } + const auto byte = static_cast<uint8_t>(s[i]); - // control characters that need \uxxxx escaping - case 0x00: - case 0x01: - case 0x02: - case 0x03: - case 0x04: - case 0x05: - case 0x06: - case 0x07: - case 0x0B: - case 0x0E: - case 0x0F: - case 0x10: - case 0x11: - case 0x12: - case 0x13: - case 0x14: - case 0x15: - case 0x16: - case 0x17: - case 0x18: - case 0x19: - case 0x1A: - case 0x1B: - case 0x1C: - case 0x1D: - case 0x1E: - case 0x1F: - { - // from c (1 byte) to \uxxxx (6 bytes) - res += 5; - break; - } - - default: + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point { - if (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F)) + switch (codepoint) { - const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - // invalid characters will be detected by throw_if_invalid_utf8 - assert (bytes != std::string::npos); - - if (bytes == 3) + case 0x08: // backspace { - // codepoints that need 4 bytes (i.e., 3 additional - // bytes) in UTF-8 need a surrogate pair when \u - // escaping is used: from 4 bytes to \uxxxx\uxxxx - // (12 bytes) - res += (12 - bytes - 1); + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; } - else + + case 0x09: // horizontal tab { - // from x bytes to \uxxxx (6 bytes) - res += (6 - bytes - 1); + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; } - // skip the additional bytes - i += bytes; - } - break; - } - } - } - - return res; - } - - static void escape_codepoint(int codepoint, string_t& result, std::size_t& pos) - { - // expecting a proper codepoint - assert(0x00 <= codepoint and codepoint <= 0x10FFFF); - - // the last written character was the backslash before the 'u' - assert(result[pos] == '\\'); - - // write the 'u' - result[++pos] = 'u'; - - // convert a number 0..15 to its hex representation (0..f) - static const std::array<char, 16> hexify = - { - { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' - } - }; - - if (codepoint < 0x10000) - { - // codepoints U+0000..U+FFFF can be represented as \uxxxx. - result[++pos] = hexify[(codepoint >> 12) & 0x0F]; - result[++pos] = hexify[(codepoint >> 8) & 0x0F]; - result[++pos] = hexify[(codepoint >> 4) & 0x0F]; - result[++pos] = hexify[codepoint & 0x0F]; - } - else - { - // codepoints U+10000..U+10FFFF need a surrogate pair to be - // represented as \uxxxx\uxxxx. - // http://www.unicode.org/faq/utf_bom.html#utf16-4 - codepoint -= 0x10000; - const int high_surrogate = 0xD800 | ((codepoint >> 10) & 0x3FF); - const int low_surrogate = 0xDC00 | (codepoint & 0x3FF); - result[++pos] = hexify[(high_surrogate >> 12) & 0x0F]; - result[++pos] = hexify[(high_surrogate >> 8) & 0x0F]; - result[++pos] = hexify[(high_surrogate >> 4) & 0x0F]; - result[++pos] = hexify[high_surrogate & 0x0F]; - ++pos; // backslash is already in output - result[++pos] = 'u'; - result[++pos] = hexify[(low_surrogate >> 12) & 0x0F]; - result[++pos] = hexify[(low_surrogate >> 8) & 0x0F]; - result[++pos] = hexify[(low_surrogate >> 4) & 0x0F]; - result[++pos] = hexify[low_surrogate & 0x0F]; - } - - ++pos; - } + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } - /*! - @brief dump escaped string + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } - Escape a string by replacing certain special characters by a sequence of an - escape character (backslash) and another character and other control - characters by a sequence of "\u" followed by a four-digit hex - representation. The escaped string is written to output stream @a o. + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } - @param[in] s the string to escape - @param[in] ensure_ascii whether to escape non-ASCII characters with - \uXXXX sequences + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } - @complexity Linear in the length of string @a s. - */ - void dump_escaped(const string_t& s, const bool ensure_ascii) const - { - throw_if_invalid_utf8(s); + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } - const auto space = extra_space(s, ensure_ascii); - if (space == 0) - { - o->write_characters(s.c_str(), s.size()); - return; - } + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast<uint16_t>(codepoint)); + bytes += 6; + } + else + { + (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)), + static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } - // create a result string of necessary size - string_t result(s.size() + space, '\\'); - std::size_t pos = 0; + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } - for (std::size_t i = 0; i < s.size(); ++i) - { - switch (s[i]) - { - case '"': // quotation mark (0x22) - { - result[pos + 1] = '"'; - pos += 2; + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; break; } - case '\\': // reverse solidus (0x5C) + case UTF8_REJECT: // decode found invalid UTF-8 byte { - // nothing to change - pos += 2; - break; - } + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + } - case '\b': // backspace (0x08) - { - result[pos + 1] = 'b'; - pos += 2; - break; - } + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } - case '\f': // formfeed (0x0C) - { - result[pos + 1] = 'f'; - pos += 2; + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD'); + } + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + } break; } - case '\n': // newline (0x0A) + default: // decode found yet incomplete multi-byte code point { - result[pos + 1] = 'n'; - pos += 2; + if (not ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; break; } + } + } - case '\r': // carriage return (0x0D) + // we finished processing the string + if (JSON_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: { - result[pos + 1] = 'r'; - pos += 2; - break; + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); } - case '\t': // horizontal tab (0x09) + case error_handler_t::ignore: { - result[pos + 1] = 't'; - pos += 2; + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); break; } - default: + case error_handler_t::replace: { - // escape control characters (0x00..0x1F) or, if - // ensure_ascii parameter is used, non-ASCII characters - if ((0x00 <= s[i] and s[i] <= 0x1F) or - (ensure_ascii and (s[i] & 0x80 or s[i] == 0x7F))) + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) { - const auto bytes = bytes_following(static_cast<uint8_t>(s[i])); - // invalid characters will be detected by throw_if_invalid_utf8 - assert (bytes != std::string::npos); - - // check that the additional bytes are present - assert(i + bytes < s.size()); - - // to use \uxxxx escaping, we first need to calculate - // the codepoint from the UTF-8 bytes - int codepoint = 0; - - // bytes is unsigned type: - assert(bytes <= 3); - switch (bytes) - { - case 0: - { - codepoint = s[i] & 0xFF; - break; - } - - case 1: - { - codepoint = ((s[i] & 0x3F) << 6) - + (s[i + 1] & 0x7F); - break; - } - - case 2: - { - codepoint = ((s[i] & 0x1F) << 12) - + ((s[i + 1] & 0x7F) << 6) - + (s[i + 2] & 0x7F); - break; - } - - case 3: - { - codepoint = ((s[i] & 0xF) << 18) - + ((s[i + 1] & 0x7F) << 12) - + ((s[i + 2] & 0x7F) << 6) - + (s[i + 3] & 0x7F); - break; - } - - default: - break; // LCOV_EXCL_LINE - } - - escape_codepoint(codepoint, result, pos); - i += bytes; + o->write_characters("\\ufffd", 6); } else { - // all other characters are added as-is - result[pos++] = s[i]; + o->write_characters("\xEF\xBF\xBD", 3); } break; } } } - - assert(pos == result.size()); - o->write_characters(result.c_str(), result.size()); } /*! @@ -6644,7 +11432,7 @@ class serializer return; } - const bool is_negative = (x <= 0) and (x != 0); // see issue #755 + const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0); // see issue #755 std::size_t i = 0; while (x != 0) @@ -6679,17 +11467,39 @@ class serializer void dump_float(number_float_t x) { // NaN / inf - if (not std::isfinite(x) or std::isnan(x)) + if (not std::isfinite(x)) { o->write_characters("null", 4); return; } - // get number of digits for a text -> float -> text round-trip - static constexpr auto d = std::numeric_limits<number_float_t>::digits10; + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if <long double> == <double>. + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or + (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024); + + dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + char* begin = number_buffer.data(); + char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast<size_t>(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10; // the actual conversion - std::ptrdiff_t len = snprintf(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); // negative value indicates an error assert(len > 0); @@ -6744,15 +11554,16 @@ class serializer followed. @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) @param[in] byte next byte to decode + @return new state - @note The function has been edited: a std::array is used and the code - point is not calculated. + @note The function has been edited: a std::array is used. @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ */ - static void decode(uint8_t& state, const uint8_t byte) + static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept { static const std::array<uint8_t, 400> utf8d = { @@ -6775,42 +11586,13 @@ class serializer }; const uint8_t type = utf8d[byte]; - state = utf8d[256u + state * 16u + type]; - } - /*! - @brief throw an exception if a string is not UTF-8 encoded - - @param[in] str UTF-8 string to check - @throw type_error.316 if passed string is not UTF-8 encoded + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6) + : static_cast<uint32_t>(0xff >> type) & (byte); - @since version 3.0.0 - */ - static void throw_if_invalid_utf8(const std::string& str) - { - // start with state 0 (= accept) - uint8_t state = 0; - - for (size_t i = 0; i < str.size(); ++i) - { - const auto byte = static_cast<uint8_t>(str[i]); - decode(state, byte); - if (state == 1) - { - // state 1 means reject - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(byte); - JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + ss.str())); - } - } - - if (state != 0) - { - // we finish reading, but do not accept: string was incomplete - std::stringstream ss; - ss << std::setw(2) << std::uppercase << std::setfill('0') << std::hex << static_cast<int>(static_cast<uint8_t>(str.back())); - JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + ss.str())); - } + state = utf8d[256u + state * 16u + type]; + return state; } private: @@ -6827,13 +11609,33 @@ class serializer /// the locale's decimal point character const char decimal_point = '\0'; + /// string buffer + std::array<char, 512> string_buffer{{}}; + /// the indentation character const char indent_char; - /// the indentation string string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; }; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/json_ref.hpp> + +#include <initializer_list> +#include <utility> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ template<typename BasicJsonType> class json_ref { @@ -6852,15 +11654,19 @@ class json_ref : owned_value(init), value_ref(&owned_value), is_rvalue(true) {} - template<class... Args> - json_ref(Args&& ... args) - : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), is_rvalue(true) - {} + template < + class... Args, + enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), + is_rvalue(true) {} // class should be movable only json_ref(json_ref&&) = default; json_ref(const json_ref&) = delete; json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; value_type moved_or_copied() const { @@ -6886,74 +11692,30 @@ class json_ref value_type* value_ref = nullptr; const bool is_rvalue; }; +} // namespace detail +} // namespace nlohmann -} // namespace detail - -/// namespace to hold default `to_json` / `from_json` functions -namespace -{ -constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; -constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; -} - - -/*! -@brief default JSONSerializer template argument - -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](http://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template<typename, typename> -struct adl_serializer -{ - /*! - @brief convert a JSON value to any value type - - This function is usually called by the `get()` function of the - @ref basic_json class (either explicit or via conversion operators). - - @param[in] j JSON value to read from - @param[in,out] val value to write to - */ - template<typename BasicJsonType, typename ValueType> - static void from_json(BasicJsonType&& j, ValueType& val) noexcept( - noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) - { - ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); - } +// #include <nlohmann/detail/json_pointer.hpp> - /*! - @brief convert any value type to a JSON value - This function is usually called by the constructors of the @ref basic_json - class. +#include <cassert> // assert +#include <numeric> // accumulate +#include <string> // string +#include <vector> // vector - @param[in,out] j JSON value to write to - @param[in] val value to read from - */ - template<typename BasicJsonType, typename ValueType> - static void to_json(BasicJsonType& j, ValueType&& val) noexcept( - noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) - { - ::nlohmann::to_json(j, std::forward<ValueType>(val)); - } -}; +// #include <nlohmann/detail/macro_scope.hpp> -/*! -@brief JSON Pointer +// #include <nlohmann/detail/exceptions.hpp> -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. +// #include <nlohmann/detail/value_t.hpp> -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) -@since version 2.0.0 -*/ +namespace nlohmann +{ +template<typename BasicJsonType> class json_pointer { - /// allow basic_json to access private members + // allow basic_json to access private members NLOHMANN_BASIC_JSON_TPL_DECLARATION friend class basic_json; @@ -6967,19 +11729,21 @@ class json_pointer @param[in] s string representing the JSON pointer; if omitted, the empty string is assumed which references the whole JSON value - @throw parse_error.107 if the given JSON pointer @a s is nonempty and - does not begin with a slash (`/`); see example below + @throw parse_error.107 if the given JSON pointer @a s is nonempty and does + not begin with a slash (`/`); see example below - @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s - is not followed by `0` (representing `~`) or `1` (representing `/`); - see example below + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is + not followed by `0` (representing `~`) or `1` (representing `/`); see + example below - @liveexample{The example shows the construction several valid JSON - pointers as well as the exceptional behavior.,json_pointer} + @liveexample{The example shows the construction several valid JSON pointers + as well as the exceptional behavior.,json_pointer} @since version 2.0.0 */ - explicit json_pointer(const std::string& s = "") : reference_tokens(split(s)) {} + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} /*! @brief return a string representation of the JSON pointer @@ -6996,7 +11760,7 @@ class json_pointer @since version 2.0.0 */ - std::string to_string() const noexcept + std::string to_string() const { return std::accumulate(reference_tokens.begin(), reference_tokens.end(), std::string{}, @@ -7021,7 +11785,7 @@ class json_pointer */ static int array_index(const std::string& s) { - size_t processed_chars = 0; + std::size_t processed_chars = 0; const int res = std::stoi(s, &processed_chars); // check if the string was completely read @@ -7051,7 +11815,7 @@ class json_pointer } /// return whether pointer points to the root document - bool is_root() const + bool is_root() const noexcept { return reference_tokens.empty(); } @@ -7076,8 +11840,66 @@ class json_pointer @throw parse_error.109 if array index is not a number @throw type_error.313 if value cannot be unflattened */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const; + BasicJsonType& get_and_create(BasicJsonType& j) const + { + using size_type = typename BasicJsonType::size_type; + auto result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + JSON_TRY + { + result = &result->operator[](static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); + } + } + + return *result; + } /*! @brief return a reference to the pointed to value @@ -7098,8 +11920,75 @@ class json_pointer @throw parse_error.109 if an array index was not a number @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const; + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->m_type == detail::value_t::null) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const char x) + { + return (x >= '0' and x <= '9'); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums or reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @throw parse_error.106 if an array index begins with '0' @@ -7107,8 +11996,57 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - NLOHMANN_BASIC_JSON_TPL& get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const; + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @brief return a const reference to the pointed to value @@ -7123,8 +12061,58 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - const NLOHMANN_BASIC_JSON_TPL& get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const; + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // use unchecked array access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @throw parse_error.106 if an array index begins with '0' @@ -7132,8 +12120,57 @@ class json_pointer @throw out_of_range.402 if the array index '-' is used @throw out_of_range.404 if the JSON pointer can not be resolved */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - const NLOHMANN_BASIC_JSON_TPL& get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const; + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } /*! @brief split the string input to reference tokens @@ -7170,11 +12207,11 @@ class json_pointer std::size_t slash = reference_string.find_first_of('/', 1), // set the beginning of the first reference token start = 1; - // we can stop if start == string::npos+1 = 0 + // we can stop if start == 0 (if slash == std::string::npos) start != 0; // set the beginning of the next reference token // (will eventually be 0 if slash == std::string::npos) - start = slash + 1, + start = (slash == std::string::npos) ? 0 : slash + 1, // find next slash slash = reference_string.find_first_of('/', start)) { @@ -7230,7 +12267,7 @@ class json_pointer {} } - /// escape "~"" to "~0" and "/" to "~1" + /// escape "~" to "~0" and "/" to "~1" static std::string escape(std::string s) { replace_substring(s, "~", "~0"); @@ -7252,10 +12289,57 @@ class json_pointer @note Empty objects or arrays are flattened to `null`. */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION static void flatten(const std::string& reference_string, - const NLOHMANN_BASIC_JSON_TPL& value, - NLOHMANN_BASIC_JSON_TPL& result); + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.m_type) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } /*! @param[in] value flattened JSON @@ -7267,19 +12351,112 @@ class json_pointer @throw type_error.315 if object values are not primitive @throw type_error.313 if value cannot be unflattened */ - NLOHMANN_BASIC_JSON_TPL_DECLARATION - static NLOHMANN_BASIC_JSON_TPL - unflatten(const NLOHMANN_BASIC_JSON_TPL& value); + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_UNLIKELY(not value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_UNLIKELY(not element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } friend bool operator==(json_pointer const& lhs, - json_pointer const& rhs) noexcept; + json_pointer const& rhs) noexcept + { + return (lhs.reference_tokens == rhs.reference_tokens); + } friend bool operator!=(json_pointer const& lhs, - json_pointer const& rhs) noexcept; + json_pointer const& rhs) noexcept + { + return not (lhs == rhs); + } /// the reference tokens std::vector<std::string> reference_tokens; }; +} // namespace nlohmann + +// #include <nlohmann/adl_serializer.hpp> + + +#include <utility> + +// #include <nlohmann/detail/conversions/from_json.hpp> + +// #include <nlohmann/detail/conversions/to_json.hpp> + + +namespace nlohmann +{ + +template<typename, typename> +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template<typename BasicJsonType, typename ValueType> + static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) + -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void()) + { + ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template <typename BasicJsonType, typename ValueType> + static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) + -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void()) + { + ::nlohmann::to_json(j, std::forward<ValueType>(val)); + } +}; + +} // namespace nlohmann + + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ /*! @brief a class to store JSON values @@ -7305,42 +12482,42 @@ and `from_json()` (@ref adl_serializer by default) @requirement The class satisfies the following concept requirements: - Basic - - [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible): + - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible): JSON values can be default constructed. The result will be a JSON null value. - - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible): + - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible): A JSON value can be constructed from an rvalue argument. - - [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible): + - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible): A JSON value can be copy-constructed from an lvalue expression. - - [MoveAssignable](http://en.cppreference.com/w/cpp/concept/MoveAssignable): + - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable): A JSON value van be assigned from an rvalue argument. - - [CopyAssignable](http://en.cppreference.com/w/cpp/concept/CopyAssignable): + - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable): A JSON value can be copy-assigned from an lvalue expression. - - [Destructible](http://en.cppreference.com/w/cpp/concept/Destructible): + - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible): JSON values can be destructed. - Layout - - [StandardLayoutType](http://en.cppreference.com/w/cpp/concept/StandardLayoutType): + - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType): JSON values have - [standard layout](http://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout): All non-static data members are private and standard layout types, the class has no virtual functions or (virtual) base classes. - Library-wide - - [EqualityComparable](http://en.cppreference.com/w/cpp/concept/EqualityComparable): + - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable): JSON values can be compared with `==`, see @ref operator==(const_reference,const_reference). - - [LessThanComparable](http://en.cppreference.com/w/cpp/concept/LessThanComparable): + - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable): JSON values can be compared with `<`, see @ref operator<(const_reference,const_reference). - - [Swappable](http://en.cppreference.com/w/cpp/concept/Swappable): + - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable): Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of other compatible types, using unqualified function call @ref swap(). - - [NullablePointer](http://en.cppreference.com/w/cpp/concept/NullablePointer): + - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer): JSON values can be compared against `std::nullptr_t` objects which are used to model the `null` value. - Container - - [Container](http://en.cppreference.com/w/cpp/concept/Container): + - [Container](https://en.cppreference.com/w/cpp/named_req/Container): JSON values can be used like STL containers and provide iterator access. - - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer); + - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer); JSON values can be used like STL containers and provide reverse iterator access. @@ -7367,15 +12544,19 @@ class basic_json { private: template<detail::value_t> friend struct detail::external_constructor; - friend ::nlohmann::json_pointer; + friend ::nlohmann::json_pointer<basic_json>; friend ::nlohmann::detail::parser<basic_json>; friend ::nlohmann::detail::serializer<basic_json>; template<typename BasicJsonType> friend class ::nlohmann::detail::iter_impl; template<typename BasicJsonType, typename CharType> friend class ::nlohmann::detail::binary_writer; - template<typename BasicJsonType> + template<typename BasicJsonType, typename SAX> friend class ::nlohmann::detail::binary_reader; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_parser; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_callback_parser; /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; @@ -7403,13 +12584,19 @@ class basic_json public: using value_t = detail::value_t; - /// @copydoc nlohmann::json_pointer - using json_pointer = ::nlohmann::json_pointer; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer<basic_json>; template<typename T, typename SFINAE> using json_serializer = JSONSerializer<T, SFINAE>; + /// how to treat decoding errors + using error_handler_t = detail::error_handler_t; /// helper type for initializer lists of basic_json values using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax<basic_json>; + //////////////// // exceptions // //////////////// @@ -7517,10 +12704,13 @@ class basic_json result["copyright"] = "(C) 2013-2017 Niels Lohmann"; result["name"] = "JSON for Modern C++"; result["url"] = "https://github.com/nlohmann/json"; - result["version"] = - { - {"string", "3.0.1"}, {"major", 3}, {"minor", 0}, {"patch", 1} - }; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; #ifdef _WIN32 result["platform"] = "win32"; @@ -7622,10 +12812,10 @@ class basic_json - When all names are unique, objects will be interoperable in the sense that all software implementations receiving that object will agree on the name-value mappings. - - When the names within an object are not unique, later stored name/value - pairs overwrite previously stored name/value pairs, leaving the used - names unique. For instance, `{"key": 1}` and `{"key": 2, "key": 1}` will - be treated as equal and both stored as `{"key": 1}`. + - When the names within an object are not unique, it is unspecified which + one of the values for a given key will be chosen. For instance, + `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or + `{"key": 2}`. - Internally, name/value pairs are stored in lexicographical order of the names. Objects will also be serialized (see @ref dump) in this order. For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored @@ -8139,7 +13329,7 @@ class basic_json object = nullptr; // silence warning, see #821 if (JSON_UNLIKELY(t == value_t::null)) { - JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.0.1")); // LCOV_EXCL_LINE + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE } break; } @@ -8182,7 +13372,7 @@ class basic_json array = create<array_t>(std::move(value)); } - void destroy(value_t t) + void destroy(value_t t) noexcept { switch (t) { @@ -8227,7 +13417,7 @@ class basic_json value is changed, because the invariant expresses a relationship between @a m_type and @a m_value. */ - void assert_invariant() const + void assert_invariant() const noexcept { assert(m_type != value_t::object or m_value.object != nullptr); assert(m_type != value_t::array or m_value.array != nullptr); @@ -8307,7 +13497,6 @@ class basic_json */ using parser_callback_t = typename parser::parser_callback_t; - ////////////////// // constructors // ////////////////// @@ -8409,6 +13598,7 @@ class basic_json - @a CompatibleType is not derived from `std::istream`, - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move constructors), + - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments) - @a CompatibleType is not a @ref basic_json nested type (e.g., @ref json_pointer, @ref iterator, etc ...) - @ref @ref json_serializer<U> has a @@ -8432,21 +13622,91 @@ class basic_json @since version 2.1.0 */ - template<typename CompatibleType, typename U = detail::uncvref_t<CompatibleType>, - detail::enable_if_t<not std::is_base_of<std::istream, U>::value and - not std::is_same<U, basic_json_t>::value and - not detail::is_basic_json_nested_type< - basic_json_t, U>::value and - detail::has_to_json<basic_json, U>::value, - int> = 0> - basic_json(CompatibleType && val) noexcept(noexcept(JSONSerializer<U>::to_json( - std::declval<basic_json_t&>(), std::forward<CompatibleType>(val)))) + template <typename CompatibleType, + typename U = detail::uncvref_t<CompatibleType>, + detail::enable_if_t< + not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept( + JSONSerializer<U>::to_json(std::declval<basic_json_t&>(), + std::forward<CompatibleType>(val)))) { JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val)); assert_invariant(); } /*! + @brief create a JSON value from an existing one + + This is a constructor for existing @ref basic_json types. + It does not hijack copy/move constructors, since the parameter has different + template arguments than the current ones. + + The constructor tries to convert the internal @ref m_value of the parameter. + + @tparam BasicJsonType a type such that: + - @a BasicJsonType is a @ref basic_json type. + - @a BasicJsonType has different template arguments than @ref basic_json_t. + + @param[in] val the @ref basic_json value to be converted. + + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. + + @exceptionsafety Depends on the called constructor. For types directly + supported by the library (i.e., all types for which no `to_json()` function + was provided), strong guarantee holds: if an exception is thrown, there are + no changes to any JSON value. + + @since version 3.2.0 + */ + template <typename BasicJsonType, + detail::enable_if_t< + detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0> + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>()); + break; + case value_t::number_float: + JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>()); + break; + case value_t::number_integer: + JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>()); + break; + case value_t::number_unsigned: + JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>()); + break; + case value_t::string: + JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>()); + break; + case value_t::object: + JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>()); + break; + case value_t::array: + JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + } + assert_invariant(); + } + + /*! @brief create a container (array or object) from an initializer list Creates a JSON value of type array or object from the passed initializer @@ -8718,7 +13978,7 @@ class basic_json @warning A precondition is enforced with a runtime assertion that will result in calling `std::abort` if this precondition is not met. Assertions can be disabled by defining `NDEBUG` at compile time. - See http://en.cppreference.com/w/cpp/error/assert for more + See https://en.cppreference.com/w/cpp/error/assert for more information. @throw invalid_iterator.201 if iterators @a first and @a last are not @@ -8858,7 +14118,7 @@ class basic_json changes to any JSON value. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. - As postcondition, it holds: `other == basic_json(other)`. @@ -8943,7 +14203,7 @@ class basic_json exceptions. @requirement This function helps `basic_json` satisfying the - [MoveConstructible](http://en.cppreference.com/w/cpp/concept/MoveConstructible) + [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible) requirements. @liveexample{The code below shows the move constructor explicitly called @@ -8977,7 +14237,7 @@ class basic_json @complexity Linear. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. @@ -8988,7 +14248,7 @@ class basic_json @since version 1.0.0 */ - reference& operator=(basic_json other) noexcept ( + basic_json& operator=(basic_json other) noexcept ( std::is_nothrow_move_constructible<value_t>::value and std::is_nothrow_move_assignable<value_t>::value and std::is_nothrow_move_constructible<json_value>::value and @@ -9014,14 +14274,14 @@ class basic_json @complexity Linear. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is linear. - All stored elements are destroyed and all memory is freed. @since version 1.0.0 */ - ~basic_json() + ~basic_json() noexcept { assert_invariant(); m_value.destroy(m_type); @@ -9054,6 +14314,10 @@ class basic_json @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters in the output are escaped with `\uXXXX` sequences, and the result consists of ASCII characters only. + @param[in] error_handler how to react on decoding errors; there are three + possible values: `strict` (throws and exception in case a decoding error + occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD), + and `ignore` (ignore invalid UTF-8 sequences during serialization). @return string containing the serialization of the JSON value @@ -9072,13 +14336,16 @@ class basic_json @see https://docs.python.org/2/library/json.html#json.dump @since version 1.0.0; indentation character @a indent_char, option - @a ensure_ascii and exceptions added in version 3.0.0 + @a ensure_ascii and exceptions added in version 3.0.0; error + handlers added in version 3.4.0. */ - string_t dump(const int indent = -1, const char indent_char = ' ', - const bool ensure_ascii = false) const + string_t dump(const int indent = -1, + const char indent_char = ' ', + const bool ensure_ascii = false, + const error_handler_t error_handler = error_handler_t::strict) const { string_t result; - serializer s(detail::output_adapter<char>(result), indent_char); + serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler); if (indent >= 0) { @@ -9619,11 +14886,34 @@ class basic_json } /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @tparam BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template<typename BasicJsonType, detail::enable_if_t< + not std::is_same<BasicJsonType, basic_json>::value and + detail::is_basic_json<BasicJsonType>::value, int> = 0> + BasicJsonType get() const + { + return *this; + } + + /*! @brief get a value (explicit) Explicit type conversion between the JSON value and a compatible value - which is [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) - and [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). The value is converted by calling the @ref json_serializer<ValueType> `from_json()` method. @@ -9659,7 +14949,7 @@ class basic_json */ template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, detail::enable_if_t < - not std::is_same<basic_json_t, ValueType>::value and + not detail::is_basic_json<ValueType>::value and detail::has_from_json<basic_json_t, ValueType>::value and not detail::has_non_default_from_json<basic_json_t, ValueType>::value, int> = 0> @@ -9683,8 +14973,8 @@ class basic_json @brief get a value (explicit); special case Explicit type conversion between the JSON value and a compatible value - which is **not** [CopyConstructible](http://en.cppreference.com/w/cpp/concept/CopyConstructible) - and **not** [DefaultConstructible](http://en.cppreference.com/w/cpp/concept/DefaultConstructible). + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). The value is converted by calling the @ref json_serializer<ValueType> `from_json()` method. @@ -9723,17 +15013,64 @@ class basic_json } /*! - @brief get a pointer value (explicit) + @brief get a value (explicit) - Explicit pointer access to the internally stored JSON value. No copies are + Explicit type conversion between the JSON value and a compatible value. + The value is filled into the input parameter by calling the @ref json_serializer<ValueType> + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType v; + JSONSerializer<ValueType>::from_json(*this, v); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer<ValueType> has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + + @tparam ValueType the input parameter type. + + @return the input parameter, allowing chaining calls. + + @throw what @ref json_serializer<ValueType> `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,get_to} + + @since version 3.3.0 + */ + template<typename ValueType, + detail::enable_if_t < + not detail::is_basic_json<ValueType>::value and + detail::has_from_json<basic_json_t, ValueType>::value, + int> = 0> + ValueType & get_to(ValueType& v) const noexcept(noexcept( + JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v))) + { + JSONSerializer<ValueType>::from_json(*this, v); + return v; + } + + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are made. - @warning The pointer becomes invalid if the underlying JSON object - changes. + @warning Writing data to the pointee of the result yields an undefined + state. @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. @return pointer to the internally stored JSON value if the requested pointer type @a PointerType fits to the JSON value; `nullptr` otherwise @@ -9743,45 +15080,43 @@ class basic_json @liveexample{The example below shows how pointers to internal values of a JSON value can be requested. Note that no type conversions are made and a `nullptr` is returned if the value and the requested pointer type does not - match.,get__PointerType} - - @sa @ref get_ptr() for explicit pointer-member access + match.,get_ptr} @since version 1.0.0 */ template<typename PointerType, typename std::enable_if< std::is_pointer<PointerType>::value, int>::type = 0> - PointerType get() noexcept + auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>())) { - // delegate the call to get_ptr - return get_ptr<PointerType>(); + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast<PointerType>(nullptr)); } /*! - @brief get a pointer value (explicit) - @copydoc get() + @brief get a pointer value (implicit) + @copydoc get_ptr() */ template<typename PointerType, typename std::enable_if< - std::is_pointer<PointerType>::value, int>::type = 0> - constexpr const PointerType get() const noexcept + std::is_pointer<PointerType>::value and + std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0> + constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>())) { - // delegate the call to get_ptr - return get_ptr<PointerType>(); + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast<PointerType>(nullptr)); } /*! - @brief get a pointer value (implicit) + @brief get a pointer value (explicit) - Implicit pointer access to the internally stored JSON value. No copies are + Explicit pointer access to the internally stored JSON value. No copies are made. - @warning Writing data to the pointee of the result yields an undefined - state. + @warning The pointer becomes invalid if the underlying JSON object + changes. @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, - @ref number_unsigned_t, or @ref number_float_t. Enforced by a static - assertion. + @ref number_unsigned_t, or @ref number_float_t. @return pointer to the internally stored JSON value if the requested pointer type @a PointerType fits to the JSON value; `nullptr` otherwise @@ -9791,59 +15126,30 @@ class basic_json @liveexample{The example below shows how pointers to internal values of a JSON value can be requested. Note that no type conversions are made and a `nullptr` is returned if the value and the requested pointer type does not - match.,get_ptr} + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access @since version 1.0.0 */ template<typename PointerType, typename std::enable_if< std::is_pointer<PointerType>::value, int>::type = 0> - PointerType get_ptr() noexcept + auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>()) { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const<typename - std::remove_pointer<typename - std::remove_const<PointerType>::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same<object_t, pointee_t>::value - or std::is_same<array_t, pointee_t>::value - or std::is_same<string_t, pointee_t>::value - or std::is_same<boolean_t, pointee_t>::value - or std::is_same<number_integer_t, pointee_t>::value - or std::is_same<number_unsigned_t, pointee_t>::value - or std::is_same<number_float_t, pointee_t>::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() - return get_impl_ptr(static_cast<PointerType>(nullptr)); + // delegate the call to get_ptr + return get_ptr<PointerType>(); } /*! - @brief get a pointer value (implicit) - @copydoc get_ptr() + @brief get a pointer value (explicit) + @copydoc get() */ template<typename PointerType, typename std::enable_if< - std::is_pointer<PointerType>::value and - std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0> - constexpr const PointerType get_ptr() const noexcept + std::is_pointer<PointerType>::value, int>::type = 0> + constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>()) { - // get the type of the PointerType (remove pointer and const) - using pointee_t = typename std::remove_const<typename - std::remove_pointer<typename - std::remove_const<PointerType>::type>::type>::type; - // make sure the type matches the allowed types - static_assert( - std::is_same<object_t, pointee_t>::value - or std::is_same<array_t, pointee_t>::value - or std::is_same<string_t, pointee_t>::value - or std::is_same<boolean_t, pointee_t>::value - or std::is_same<number_integer_t, pointee_t>::value - or std::is_same<number_unsigned_t, pointee_t>::value - or std::is_same<number_float_t, pointee_t>::value - , "incompatible pointer type"); - - // delegate the call to get_impl_ptr<>() const - return get_impl_ptr(static_cast<PointerType>(nullptr)); + // delegate the call to get_ptr + return get_ptr<PointerType>(); } /*! @@ -9925,13 +15231,16 @@ class basic_json template < typename ValueType, typename std::enable_if < not std::is_pointer<ValueType>::value and not std::is_same<ValueType, detail::json_ref<basic_json>>::value and - not std::is_same<ValueType, typename string_t::value_type>::value + not std::is_same<ValueType, typename string_t::value_type>::value and + not detail::is_basic_json<ValueType>::value + #ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value -#endif -#if defined(JSON_HAS_CPP_17) +#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914 and not std::is_same<ValueType, typename std::string_view>::value #endif +#endif + and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value , int >::type = 0 > operator ValueType() const { @@ -10195,7 +15504,7 @@ class basic_json return m_value.array->operator[](idx); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()))); } /*! @@ -10225,7 +15534,7 @@ class basic_json return m_value.array->operator[](idx); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()))); } /*! @@ -10271,7 +15580,7 @@ class basic_json return m_value.object->operator[](key); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10313,7 +15622,7 @@ class basic_json return m_value.object->find(key)->second; } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10360,7 +15669,7 @@ class basic_json return m_value.object->operator[](key); } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10403,7 +15712,7 @@ class basic_json return m_value.object->find(key)->second; } - JSON_THROW(type_error::create(305, "cannot use operator[] with " + std::string(type_name()))); + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); } /*! @@ -10476,7 +15785,7 @@ class basic_json /*! @brief overload for a default value of type const char* - @copydoc basic_json::value(const typename object_t::key_type&, ValueType) const + @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const */ string_t value(const typename object_t::key_type& key, const char* default_value) const { @@ -10512,7 +15821,7 @@ class basic_json @return copy of the element at key @a key or @a default_value if @a key is not found - @throw type_error.306 if the JSON value is not an objec; in that case, + @throw type_error.306 if the JSON value is not an object; in that case, using `value()` with a key makes no sense. @complexity Logarithmic in the size of the container. @@ -10536,7 +15845,7 @@ class basic_json { return ptr.get_checked(this); } - JSON_CATCH (out_of_range&) + JSON_INTERNAL_CATCH (out_of_range&) { return default_value; } @@ -11047,7 +16356,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. @@ -11086,7 +16395,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).begin()`. @@ -11118,7 +16427,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. @@ -11157,7 +16466,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).end()`. @@ -11187,7 +16496,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `reverse_iterator(end())`. @@ -11224,7 +16533,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `reverse_iterator(begin())`. @@ -11261,7 +16570,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`. @@ -11290,7 +16599,7 @@ class basic_json @complexity Constant. @requirement This function helps `basic_json` satisfying the - [ReversibleContainer](http://en.cppreference.com/w/cpp/concept/ReversibleContainer) + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) requirements: - The complexity is constant. - Has the semantics of `const_cast<const basic_json&>(*this).rend()`. @@ -11361,18 +16670,100 @@ class basic_json @note The name of this function is not yet final and may change in the future. + + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref items() instead; + that is, replace `json::iterator_wrapper(j)` with `j.items()`. */ - static iteration_proxy<iterator> iterator_wrapper(reference ref) + JSON_DEPRECATED + static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept { - return iteration_proxy<iterator>(ref); + return ref.items(); } /*! @copydoc iterator_wrapper(reference) */ - static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) + JSON_DEPRECATED + static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept { - return iteration_proxy<const_iterator>(ref); + return ref.items(); + } + + /*! + @brief helper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + For loop without `items()` function: + + @code{cpp} + for (auto it = j_object.begin(); it != j_object.end(); ++it) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + Range-based for loop without `items()` function: + + @code{cpp} + for (auto it : j_object) + { + // "it" is of type json::reference and has no key() member + std::cout << "value: " << it << '\n'; + } + @endcode + + Range-based for loop with `items()` function: + + @code{cpp} + for (auto& el : j_object.items()) + { + std::cout << "key: " << el.key() << ", value:" << el.value() << '\n'; + } + @endcode + + The `items()` function also allows to use + [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding) + (C++17): + + @code{cpp} + for (auto& [key, val] : j_object.items()) + { + std::cout << "key: " << key << ", value:" << val << '\n'; + } + @endcode + + @note When iterating over an array, `key()` will return the index of the + element as string (see example). For primitive types (e.g., numbers), + `key()` returns an empty string. + + @return iteration proxy object wrapping @a ref with an interface to use in + range-based for loops + + @liveexample{The following code shows how the function is used.,items} + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 3.1.0, structured bindings support since 3.5.0. + */ + iteration_proxy<iterator> items() noexcept + { + return iteration_proxy<iterator>(*this); + } + + /*! + @copydoc items() + */ + iteration_proxy<const_iterator> items() const noexcept + { + return iteration_proxy<const_iterator>(*this); } /// @} @@ -11417,7 +16808,7 @@ class basic_json false in the case of a string. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `begin() == end()`. @@ -11488,7 +16879,7 @@ class basic_json the case of a string. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of `std::distance(begin(), end())`. @@ -11558,7 +16949,7 @@ class basic_json @exceptionsafety No-throw guarantee: this function never throws exceptions. @requirement This function helps `basic_json` satisfying the - [Container](http://en.cppreference.com/w/cpp/concept/Container) + [Container](https://en.cppreference.com/w/cpp/named_req/Container) requirements: - The complexity is constant. - Has the semantics of returning `b.size()` where `b` is the largest @@ -11970,6 +17361,26 @@ class basic_json return {it, res.second}; } + /// Helper for insertion of an iterator + /// @note: This uses std::distance to support GCC 4.8, + /// see https://github.com/nlohmann/json/pull/1257 + template<typename... Args> + iterator insert_iterator(const_iterator pos, Args&& ... args) + { + iterator result(this); + assert(m_value.array != nullptr); + + auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); + m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...); + result.m_it.array_iterator = m_value.array->begin() + insert_pos; + + // This could have been written as: + // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // but the return value of insert is missing in GCC 4.8, so it is written this way instead. + + return result; + } + /*! @brief inserts element @@ -12004,9 +17415,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, val); - return result; + return insert_iterator(pos, val); } JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); @@ -12057,9 +17466,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); - return result; + return insert_iterator(pos, cnt, val); } JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); @@ -12121,12 +17528,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert( - pos.m_it.array_iterator, - first.m_it.array_iterator, - last.m_it.array_iterator); - return result; + return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); } /*! @@ -12168,9 +17570,7 @@ class basic_json } // insert to array and return iterator - iterator result(this); - result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, ilist.begin(), ilist.end()); - return result; + return insert_iterator(pos, ilist.begin(), ilist.end()); } /*! @@ -12312,7 +17712,7 @@ class basic_json // passed iterators must belong to objects if (JSON_UNLIKELY(not first.m_object->is_object() - or not first.m_object->is_object())) + or not last.m_object->is_object())) { JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); } @@ -12943,8 +18343,8 @@ class basic_json /*! @brief serialize to stream - @deprecated This stream operator is deprecated and will be removed in a - future version of the library. Please use + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref operator<<(std::ostream&, const basic_json&) instead; that is, replace calls like `j >> o;` with `o << j;`. @since version 1.0.0; deprecated since version 3.0.0 @@ -12999,6 +18399,8 @@ class basic_json @param[in] cb a parser callback function of type @ref parser_callback_t which is used to control the deserialization by filtering unwanted values (optional) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) @return result of the deserialization @@ -13027,19 +18429,7 @@ class basic_json @since version 2.0.3 (contiguous containers) */ - static basic_json parse(detail::input_adapter i, - const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) - { - basic_json result; - parser(i, cb, allow_exceptions).parse(true, result); - return result; - } - - /*! - @copydoc basic_json parse(detail::input_adapter, const parser_callback_t) - */ - static basic_json parse(detail::input_adapter& i, + static basic_json parse(detail::input_adapter&& i, const parser_callback_t cb = nullptr, const bool allow_exceptions = true) { @@ -13048,14 +18438,80 @@ class basic_json return result; } - static bool accept(detail::input_adapter i) + static bool accept(detail::input_adapter&& i) { return parser(i).accept(true); } - static bool accept(detail::input_adapter& i) + /*! + @brief generate SAX events + + The SAX event lister must follow the interface of @ref json_sax. + + This function reads from a compatible input. Examples are: + - an array of 1-byte values + - strings with character/literal type with size of 1 byte + - input streams + - container with contiguous storage of 1-byte values. Compatible container + types include `std::vector`, `std::string`, `std::array`, + `std::valarray`, and `std::initializer_list`. Furthermore, C-style + arrays can be used with `std::begin()`/`std::end()`. User-defined + containers can be used as long as they implement random-access iterators + and a contiguous storage. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] i input to read from + @param[in,out] sax SAX event listener + @param[in] format the format to parse (JSON, CBOR, MessagePack, or UBJSON) + @param[in] strict whether the input has to be consumed completely + + @return return value of the last processed SAX event + + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the SAX consumer @a sax has + a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `sax_parse()` function + reading from string and processing the events with a user-defined SAX + event consumer.,sax_parse} + + @since version 3.2.0 + */ + template <typename SAX> + static bool sax_parse(detail::input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true) { - return parser(i).accept(true); + assert(sax); + switch (format) + { + case input_format_t::json: + return parser(std::move(i)).sax_parse(sax, strict); + default: + return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict); + } } /*! @@ -13127,10 +18583,19 @@ class basic_json return parser(detail::input_adapter(first, last)).accept(true); } + template<class IteratorType, class SAX, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax) + { + return parser(detail::input_adapter(first, last)).sax_parse(sax); + } + /*! @brief deserialize from stream - @deprecated This stream operator is deprecated and will be removed in a - future version of the library. Please use + @deprecated This stream operator is deprecated and will be removed in + version 4.0.0 of the library. Please use @ref operator>>(std::istream&, basic_json&) instead; that is, replace calls like `j << i;` with `i >> j;`. @since version 1.0.0; deprecated since version 3.0.0 @@ -13331,9 +18796,11 @@ class basic_json vector in CBOR format.,to_cbor} @sa http://cbor.io - @sa @ref from_cbor(const std::vector<uint8_t>&, const size_t) for the + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the analogous deserialization @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format @since version 2.0.9 */ @@ -13426,9 +18893,10 @@ class basic_json vector in MessagePack format.,to_msgpack} @sa http://msgpack.org - @sa @ref from_msgpack(const std::vector<uint8_t>&, const size_t) for the - analogous deserialization + @sa @ref from_msgpack for the analogous deserialization @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format @since version 2.0.9 */ @@ -13450,6 +18918,192 @@ class basic_json } /*! + @brief create a UBJSON serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the UBJSON + (Universal Binary JSON) serialization format. UBJSON aims to be more compact + than JSON itself, yet more efficient to parse. + + The library uses the following mapping from JSON values types to + UBJSON types according to the UBJSON specification: + + JSON value type | value/range | UBJSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | `Z` + boolean | `true` | true | `T` + boolean | `false` | false | `F` + number_integer | -9223372036854775808..-2147483649 | int64 | `L` + number_integer | -2147483648..-32769 | int32 | `l` + number_integer | -32768..-129 | int16 | `I` + number_integer | -128..127 | int8 | `i` + number_integer | 128..255 | uint8 | `U` + number_integer | 256..32767 | int16 | `I` + number_integer | 32768..2147483647 | int32 | `l` + number_integer | 2147483648..9223372036854775807 | int64 | `L` + number_unsigned | 0..127 | int8 | `i` + number_unsigned | 128..255 | uint8 | `U` + number_unsigned | 256..32767 | int16 | `I` + number_unsigned | 32768..2147483647 | int32 | `l` + number_unsigned | 2147483648..9223372036854775807 | int64 | `L` + number_float | *any value* | float64 | `D` + string | *with shortest length indicator* | string | `S` + array | *see notes on optimized format* | array | `[` + object | *see notes on optimized format* | map | `{` + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a UBJSON value. + + @note The following values can **not** be converted to a UBJSON value: + - strings with more than 9223372036854775807 bytes (theoretical) + - unsigned integer numbers above 9223372036854775807 + + @note The following markers are not used in the conversion: + - `Z`: no-op values are not created. + - `C`: single-byte strings are serialized with `S` markers. + + @note Any UBJSON output created @ref to_ubjson can be successfully parsed + by @ref from_ubjson. + + @note If NaN or Infinity are stored inside a JSON number, they are + serialized properly. This behavior differs from the @ref dump() + function which serializes NaN or Infinity to `null`. + + @note The optimized formats for containers are supported: Parameter + @a use_size adds size information to the beginning of a container and + removes the closing marker. Parameter @a use_type further checks + whether all elements of a container have the same type and adds the + type marker to the beginning of the container. The @a use_type + parameter must only be used together with @a use_size = true. Note + that @a use_size = true alone may result in larger representations - + the benefit of this parameter is that the receiving side is + immediately informed on the number of elements of the container. + + @param[in] j JSON value to serialize + @param[in] use_size whether to add size annotations to container types + @param[in] use_type whether to add type annotations to container types + (must be combined with @a use_size = true) + @return UBJSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in UBJSON format.,to_ubjson} + + @sa http://ubjson.org + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + analogous deserialization + @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + + @since version 3.1.0 + */ + static std::vector<uint8_t> to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector<uint8_t> result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type); + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<char> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<char>(o).write_ubjson(j, use_size, use_type); + } + + + /*! + @brief Serializes the given JSON object `j` to BSON and returns a vector + containing the corresponding BSON-representation. + + BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are + stored as a single entity (a so-called document). + + The library uses the following mapping from JSON values types to BSON types: + + JSON value type | value/range | BSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | 0x0A + boolean | `true`, `false` | boolean | 0x08 + number_integer | -9223372036854775808..-2147483649 | int64 | 0x12 + number_integer | -2147483648..2147483647 | int32 | 0x10 + number_integer | 2147483648..9223372036854775807 | int64 | 0x12 + number_unsigned | 0..2147483647 | int32 | 0x10 + number_unsigned | 2147483648..9223372036854775807 | int64 | 0x12 + number_unsigned | 9223372036854775808..18446744073709551615| -- | -- + number_float | *any value* | double | 0x01 + string | *any value* | string | 0x02 + array | *any value* | document | 0x04 + object | *any value* | document | 0x03 + + @warning The mapping is **incomplete**, since only JSON-objects (and things + contained therein) can be serialized to BSON. + Also, integers larger than 9223372036854775807 cannot be serialized to BSON, + and the keys may not contain U+0000, since they are serialized a + zero-terminated c-strings. + + @throw out_of_range.407 if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807` + @throw out_of_range.409 if a key in `j` contains a NULL (U+0000) + @throw type_error.317 if `!j.is_object()` + + @pre The input `j` is required to be an object: `j.is_object() == true`. + + @note Any BSON output created via @ref to_bson can be successfully parsed + by @ref from_bson. + + @param[in] j JSON value to serialize + @return BSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in BSON format.,to_bson} + + @sa http://bsonspec.org/spec.html + @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the + analogous deserialization + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + @sa @ref to_cbor(const basic_json&) for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + */ + static std::vector<uint8_t> to_bson(const basic_json& j) + { + std::vector<uint8_t> result; + to_bson(j, result); + return result; + } + + /*! + @brief Serializes the given JSON object `j` to BSON and forwards the + corresponding BSON-representation to the given output_adapter `o`. + @param j The JSON object to convert to BSON. + @param o The output adapter that receives the binary BSON representation. + @pre The input `j` shall be an object: `j.is_object() == true` + @sa @ref to_bson(const basic_json&) + */ + static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o) + { + binary_writer<uint8_t>(o).write_bson(j); + } + + /*! + @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>) + */ + static void to_bson(const basic_json& j, detail::output_adapter<char> o) + { + binary_writer<char>(o).write_bson(j); + } + + + /*! @brief create a JSON value from an input in CBOR format Deserializes a given input @a i to a JSON value using the CBOR (Concise @@ -13490,7 +19144,7 @@ class basic_json map | object | 0xBF False | `false` | 0xF4 True | `true` | 0xF5 - Nill | `null` | 0xF6 + Null | `null` | 0xF6 Half-Precision Float | number_float | 0xF9 Single-Precision Float | number_float | 0xFA Double-Precision Float | number_float | 0xFB @@ -13518,6 +19172,9 @@ class basic_json @param[in] i an input in CBOR format convertible to an input adapter @param[in] strict whether to expect the input to be consumed until EOF (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + @return deserialized JSON value @throw parse_error.110 if the given input ends prematurely or the end of @@ -13533,27 +19190,39 @@ class basic_json @sa http://cbor.io @sa @ref to_cbor(const basic_json&) for the analogous serialization - @sa @ref from_msgpack(detail::input_adapter, const bool) for the + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the related MessagePack format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + related UBJSON format @since version 2.0.9; parameter @a start_index since 2.1.1; changed to consume input adapters, removed start_index parameter, and added - @a strict parameter since 3.0.0 + @a strict parameter since 3.0.0; added @a allow_exceptions parameter + since 3.2.0 */ - static basic_json from_cbor(detail::input_adapter i, - const bool strict = true) + static basic_json from_cbor(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(i).parse_cbor(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! - @copydoc from_cbor(detail::input_adapter, const bool) + @copydoc from_cbor(detail::input_adapter&&, const bool, const bool) */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> - static basic_json from_cbor(A1 && a1, A2 && a2, const bool strict = true) + static basic_json from_cbor(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_cbor(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! @@ -13606,6 +19275,10 @@ class basic_json adapter @param[in] strict whether to expect the input to be consumed until EOF (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value @throw parse_error.110 if the given input ends prematurely or the end of file was not reached when @a strict was set to true @@ -13620,29 +19293,212 @@ class basic_json @sa http://msgpack.org @sa @ref to_msgpack(const basic_json&) for the analogous serialization - @sa @ref from_cbor(detail::input_adapter, const bool) for the related CBOR - format + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for + the related UBJSON format + @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for + the related BSON format @since version 2.0.9; parameter @a start_index since 2.1.1; changed to consume input adapters, removed start_index parameter, and added - @a strict parameter since 3.0.0 + @a strict parameter since 3.0.0; added @a allow_exceptions parameter + since 3.2.0 + */ + static basic_json from_msgpack(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_msgpack(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @brief create a JSON value from an input in UBJSON format + + Deserializes a given input @a i to a JSON value using the UBJSON (Universal + Binary JSON) serialization format. + + The library maps UBJSON types to JSON value types as follows: + + UBJSON type | JSON value type | marker + ----------- | --------------------------------------- | ------ + no-op | *no value, next value is read* | `N` + null | `null` | `Z` + false | `false` | `F` + true | `true` | `T` + float32 | number_float | `d` + float64 | number_float | `D` + uint8 | number_unsigned | `U` + int8 | number_integer | `i` + int16 | number_integer | `I` + int32 | number_integer | `l` + int64 | number_integer | `L` + string | string | `S` + char | string | `C` + array | array (optimized values are supported) | `[` + object | object (optimized values are supported) | `{` + + @note The mapping is **complete** in the sense that any UBJSON value can + be converted to a JSON value. + + @param[in] i an input in UBJSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.110 if the given input ends prematurely or the end of + file was not reached when @a strict was set to true + @throw parse_error.112 if a parse error occurs + @throw parse_error.113 if a string could not be parsed successfully + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + UBJSON format to a JSON value.,from_ubjson} + + @sa http://ubjson.org + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for + the related MessagePack format + @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for + the related BSON format + + @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0 */ - static basic_json from_msgpack(detail::input_adapter i, - const bool strict = true) + static basic_json from_ubjson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(i).parse_msgpack(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } /*! - @copydoc from_msgpack(detail::input_adapter, const bool) + @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool) */ template<typename A1, typename A2, detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> - static basic_json from_msgpack(A1 && a1, A2 && a2, const bool strict = true) + static basic_json from_ubjson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) { - return binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).parse_msgpack(strict); + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); } + /*! + @brief Create a JSON value from an input in BSON format + + Deserializes a given input @a i to a JSON value using the BSON (Binary JSON) + serialization format. + + The library maps BSON record types to JSON value types as follows: + + BSON type | BSON marker byte | JSON value type + --------------- | ---------------- | --------------------------- + double | 0x01 | number_float + string | 0x02 | string + document | 0x03 | object + array | 0x04 | array + binary | 0x05 | still unsupported + undefined | 0x06 | still unsupported + ObjectId | 0x07 | still unsupported + boolean | 0x08 | boolean + UTC Date-Time | 0x09 | still unsupported + null | 0x0A | null + Regular Expr. | 0x0B | still unsupported + DB Pointer | 0x0C | still unsupported + JavaScript Code | 0x0D | still unsupported + Symbol | 0x0E | still unsupported + JavaScript Code | 0x0F | still unsupported + int32 | 0x10 | number_integer + Timestamp | 0x11 | still unsupported + 128-bit decimal float | 0x13 | still unsupported + Max Key | 0x7F | still unsupported + Min Key | 0xFF | still unsupported + + @warning The mapping is **incomplete**. The unsupported mappings + are indicated in the table above. + + @param[in] i an input in BSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.114 if an unsupported BSON record type is encountered + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + BSON format to a JSON value.,from_bson} + + @sa http://bsonspec.org/spec.html + @sa @ref to_bson(const basic_json&) for the analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for + the related MessagePack format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + related UBJSON format + */ + static basic_json from_bson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_bson(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_bson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + + /// @} ////////////////////////// @@ -14008,20 +19864,20 @@ class basic_json // avoid undefined behavior JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); } - else - { - // default case: insert add offset - parent.insert(parent.begin() + static_cast<difference_type>(idx), val); - } + + // default case: insert add offset + parent.insert(parent.begin() + static_cast<difference_type>(idx), val); } break; } + // LCOV_EXCL_START default: { // if there exists a parent it cannot be primitive - assert(false); // LCOV_EXCL_LINE + assert(false); } + // LCOV_EXCL_STOP } } }; @@ -14066,7 +19922,7 @@ class basic_json // wrapper to get a value for an operation const auto get_value = [&val](const std::string & op, const std::string & member, - bool string_type) -> basic_json& + bool string_type) -> basic_json & { // find value auto it = val.m_value.object->find(member); @@ -14163,7 +20019,7 @@ class basic_json // the "path" location must exist - use at() success = (result.at(ptr) == get_value("test", "value", false)); } - JSON_CATCH (out_of_range&) + JSON_INTERNAL_CATCH (out_of_range&) { // ignore out of range errors: success remains false } @@ -14216,6 +20072,7 @@ class basic_json diff for two JSON values.,diff} @sa @ref patch -- apply a JSON patch + @sa @ref merge_patch -- apply a JSON Merge Patch @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) @@ -14347,418 +20204,86 @@ class basic_json } /// @} -}; -///////////// -// presets // -///////////// + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; - -////////////////// -// json_pointer // -////////////////// - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_and_create(NLOHMANN_BASIC_JSON_TPL& j) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - auto result = &j; - - // in case no reference tokens exist, return a reference to the JSON value - // j which will be overwritten by a primitive value - for (const auto& reference_token : reference_tokens) - { - switch (result->m_type) - { - case detail::value_t::null: - { - if (reference_token == "0") - { - // start a new array if reference token is 0 - result = &result->operator[](0); - } - else - { - // start a new object otherwise - result = &result->operator[](reference_token); - } - break; - } - - case detail::value_t::object: - { - // create an entry in the object - result = &result->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - // create an entry in the array - JSON_TRY - { - result = &result->operator[](static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } + /// @name JSON Merge Patch functions + /// @{ - /* - The following code is only reached if there exists a reference - token _and_ the current value is primitive. In this case, we have - an error situation, because primitive values may only occur as - single value; that is, with an empty list of reference tokens. - */ - default: - JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); - } - } + /*! + @brief applies a JSON Merge Patch + + The merge patch format is primarily intended for use with the HTTP PATCH + method as a means of describing a set of modifications to a target + resource's content. This function applies a merge patch to the current + JSON value. + + The function implements the following algorithm from Section 2 of + [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396): + + ``` + define MergePatch(Target, Patch): + if Patch is an Object: + if Target is not an Object: + Target = {} // Ignore the contents and set it to an empty Object + for each Name/Value pair in Patch: + if Value is null: + if Name exists in Target: + remove the Name/Value pair from Target + else: + Target[Name] = MergePatch(Target[Name], Value) + return Target + else: + return Patch + ``` + + Thereby, `Target` is the current object; that is, the patch is applied to + the current value. + + @param[in] apply_patch the patch to apply + + @complexity Linear in the lengths of @a patch. + + @liveexample{The following code shows how a JSON Merge Patch is applied to + a JSON document.,merge_patch} - return *result; -} + @sa @ref patch -- apply a JSON patch + @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_unchecked(NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) + @since version 3.0.0 + */ + void merge_patch(const basic_json& apply_patch) { - // convert null values to arrays or objects before continuing - if (ptr->m_type == detail::value_t::null) - { - // check if reference token is a number - const bool nums = - std::all_of(reference_token.begin(), reference_token.end(), - [](const char x) - { - return (x >= '0' and x <= '9'); - }); - - // change value to array for numbers or "-" or to object otherwise - *ptr = (nums or reference_token == "-") - ? detail::value_t::array - : detail::value_t::object; - } - - switch (ptr->m_type) + if (apply_patch.is_object()) { - case detail::value_t::object: + if (not is_object()) { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; + *this = object(); } - - case detail::value_t::array: + for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it) { - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - if (reference_token == "-") + if (it.value().is_null()) { - // explicitly treat "-" as index beyond the end - ptr = &ptr->operator[](ptr->m_value.array->size()); + erase(it.key()); } else { - // convert array index to number; unchecked access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_checked(NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + operator[](it.key()).merge_patch(it.value()); } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -const NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_unchecked(const NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // use unchecked object access - ptr = &ptr->operator[](reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" cannot be used for const access - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // use unchecked array access - JSON_TRY - { - ptr = &ptr->operator[]( - static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -const NLOHMANN_BASIC_JSON_TPL& -json_pointer::get_checked(const NLOHMANN_BASIC_JSON_TPL* ptr) const -{ - using size_type = typename NLOHMANN_BASIC_JSON_TPL::size_type; - for (const auto& reference_token : reference_tokens) - { - switch (ptr->m_type) - { - case detail::value_t::object: - { - // note: at performs range check - ptr = &ptr->at(reference_token); - break; - } - - case detail::value_t::array: - { - if (JSON_UNLIKELY(reference_token == "-")) - { - // "-" always fails the range check - JSON_THROW(detail::out_of_range::create(402, - "array index '-' (" + std::to_string(ptr->m_value.array->size()) + - ") is out of range")); - } - - // error condition (cf. RFC 6901, Sect. 4) - if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) - { - JSON_THROW(detail::parse_error::create(106, 0, - "array index '" + reference_token + - "' must not begin with '0'")); - } - - // note: at performs range check - JSON_TRY - { - ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); - } - JSON_CATCH(std::invalid_argument&) - { - JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); - } - break; - } - - default: - JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); - } - } - - return *ptr; -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -void json_pointer::flatten(const std::string& reference_string, - const NLOHMANN_BASIC_JSON_TPL& value, - NLOHMANN_BASIC_JSON_TPL& result) -{ - switch (value.m_type) - { - case detail::value_t::array: - { - if (value.m_value.array->empty()) - { - // flatten empty array as null - result[reference_string] = nullptr; - } - else - { - // iterate array and use index as reference string - for (std::size_t i = 0; i < value.m_value.array->size(); ++i) - { - flatten(reference_string + "/" + std::to_string(i), - value.m_value.array->operator[](i), result); - } - } - break; - } - - case detail::value_t::object: - { - if (value.m_value.object->empty()) - { - // flatten empty object as null - result[reference_string] = nullptr; - } - else - { - // iterate object and use keys as reference string - for (const auto& element : *value.m_value.object) - { - flatten(reference_string + "/" + escape(element.first), element.second, result); - } - } - break; - } - - default: - { - // add primitive value with its reference string - result[reference_string] = value; - break; - } - } -} - -NLOHMANN_BASIC_JSON_TPL_DECLARATION -NLOHMANN_BASIC_JSON_TPL -json_pointer::unflatten(const NLOHMANN_BASIC_JSON_TPL& value) -{ - if (JSON_UNLIKELY(not value.is_object())) - { - JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); - } - - NLOHMANN_BASIC_JSON_TPL result; - - // iterate the JSON object values - for (const auto& element : *value.m_value.object) - { - if (JSON_UNLIKELY(not element.second.is_primitive())) + else { - JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + *this = apply_patch; } - - // assign value to reference pointed to by JSON pointer; Note that if - // the JSON pointer is "" (i.e., points to the whole value), function - // get_and_create returns a reference to result itself. An assignment - // will then create a primitive value. - json_pointer(element.first).get_and_create(result) = element.second; } - return result; -} - -inline bool operator==(json_pointer const& lhs, json_pointer const& rhs) noexcept -{ - return (lhs.reference_tokens == rhs.reference_tokens); -} - -inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcept -{ - return not (lhs == rhs); -} + /// @} +}; } // namespace nlohmann - /////////////////////// // nonmember support // /////////////////////// @@ -14766,20 +20291,6 @@ inline bool operator!=(json_pointer const& lhs, json_pointer const& rhs) noexcep // specialization of std::swap, and std::hash namespace std { -/*! -@brief exchanges the values of two JSON objects - -@since version 1.0.0 -*/ -template<> -inline void swap(nlohmann::json& j1, - nlohmann::json& j2) noexcept( - is_nothrow_move_constructible<nlohmann::json>::value and - is_nothrow_move_assignable<nlohmann::json>::value - ) -{ - j1.swap(j2); -} /// hash value for JSON objects template<> @@ -14815,6 +20326,20 @@ struct less< ::nlohmann::detail::value_t> } }; +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template<> +inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept( + is_nothrow_move_constructible<nlohmann::json>::value and + is_nothrow_move_assignable<nlohmann::json>::value +) +{ + j1.swap(j2); +} + } // namespace std /*! @@ -14853,6 +20378,9 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std return nlohmann::json::json_pointer(std::string(s, n)); } +// #include <nlohmann/detail/macro_unscope.hpp> + + // restore GCC/clang diagnostic settings #if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) #pragma GCC diagnostic pop @@ -14862,13 +20390,17 @@ inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std #endif // clean up +#undef JSON_INTERNAL_CATCH #undef JSON_CATCH #undef JSON_THROW #undef JSON_TRY #undef JSON_LIKELY #undef JSON_UNLIKELY #undef JSON_DEPRECATED +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 #undef NLOHMANN_BASIC_JSON_TPL_DECLARATION #undef NLOHMANN_BASIC_JSON_TPL + #endif diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index 8792a4a25..f9db16268 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,6 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix -resolve-system-dependencies_LIBS := libstore libmain libutil libformat +resolve-system-dependencies_LIBS := libstore libmain libutil resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc |